def getQvar(self, quantile): quantile = quantile * 100 minus = "" if (abs(quantile - int(quantile)) > 0.01): var = "q" + minus + str(abs(quantile)).replace(".", "") else: var = "q" + minus + str(int(abs(quantile))) if (not self.hasMetric(var) and quantile == 50): Util.warning("Could not find q50, using fcst instead") return "fcst" return var
def run(argv): ############ # Defaults # ############ ifiles = list() ofile = None metric = None locations = None latlonRange = None training = 0 thresholds = None dates = None climFile = None climType = "subtract" leg = None ylabel = None xlabel = None title = None offsets = None xdim = None sdim = None figSize = None dpi = 100 showText = False showMap = False noMargin = False binType = None markerSize = None lineWidth = None tickFontSize = None labFontSize = None legFontSize = None type = "plot" XRotation = None MajorLength = None MinorLength = None MajorWidth = None Bottom = None Top = None Right = None Left = None Pad = None showPerfect = None cType = "mean" doHist = False doSort = False doAcc = False xlim = None ylim = None clim = None version = None listThresholds = False listQuantiles = False listLocations = False listDates = False # Read command line arguments i = 1 while (i < len(argv)): arg = argv[i] if (arg[0] == '-'): # Process option if (arg == "-nomargin"): noMargin = True elif (arg == "--version"): version = True elif (arg == "--list-thresholds"): listThresholds = True elif (arg == "--list-quantiles"): listQuantiles = True elif (arg == "--list-locations"): listLocations = True elif (arg == "--list-dates"): listDates = True elif (arg == "-sp"): showPerfect = True elif (arg == "-hist"): doHist = True elif (arg == "-acc"): doAcc = True elif (arg == "-sort"): doSort = True else: if (arg == "-f"): ofile = argv[i + 1] elif (arg == "-l"): locations = Util.parseNumbers(argv[i + 1]) elif (arg == "-llrange"): latlonRange = Util.parseNumbers(argv[i + 1]) elif (arg == "-t"): training = int(argv[i + 1]) elif (arg == "-x"): xdim = argv[i + 1] elif (arg == "-o"): offsets = Util.parseNumbers(argv[i + 1]) elif (arg == "-leg"): leg = unicode(argv[i + 1], 'utf8') elif (arg == "-ylabel"): ylabel = unicode(argv[i + 1], 'utf8') elif (arg == "-xlabel"): xlabel = unicode(argv[i + 1], 'utf8') elif (arg == "-title"): title = unicode(argv[i + 1], 'utf8') elif (arg == "-b"): binType = argv[i + 1] elif (arg == "-type"): type = argv[i + 1] elif (arg == "-fs"): figSize = argv[i + 1] elif (arg == "-dpi"): dpi = int(argv[i + 1]) elif (arg == "-d"): # Either format is ok: # -d 20150101 20150103 # -d 20150101:20150103 if (i + 2 < len(argv) and argv[i + 2].isdigit()): dates = Util.parseNumbers( "%s:%s" % (argv[i + 1], argv[i + 2]), True) i = i + 1 else: dates = Util.parseNumbers(argv[i + 1], True) elif (arg == "-c"): climFile = argv[i + 1] climType = "subtract" elif (arg == "-C"): climFile = argv[i + 1] climType = "divide" elif (arg == "-xlim"): xlim = Util.parseNumbers(argv[i + 1]) elif (arg == "-ylim"): ylim = Util.parseNumbers(argv[i + 1]) elif (arg == "-clim"): clim = Util.parseNumbers(argv[i + 1]) elif (arg == "-s"): sdim = argv[i + 1] elif (arg == "-ct"): cType = argv[i + 1] elif (arg == "-r"): thresholds = Util.parseNumbers(argv[i + 1]) elif (arg == "-ms"): markerSize = float(argv[i + 1]) elif (arg == "-lw"): lineWidth = float(argv[i + 1]) elif (arg == "-tickfs"): tickFontSize = float(argv[i + 1]) elif (arg == "-labfs"): labFontSize = float(argv[i + 1]) elif (arg == "-legfs"): legFontSize = float(argv[i + 1]) elif (arg == "-xrot"): XRotation = float(argv[i + 1]) elif (arg == "-majlth"): MajorLength = float(argv[i + 1]) elif (arg == "-minlth"): MinorLength = float(argv[i + 1]) elif (arg == "-majwid"): MajorWidth = float(argv[i + 1]) elif (arg == "-bot"): Bottom = float(argv[i + 1]) elif (arg == "-top"): Top = float(argv[i + 1]) elif (arg == "-right"): Right = float(argv[i + 1]) elif (arg == "-left"): Left = float(argv[i + 1]) elif (arg == "-pad"): Pad = argv[i + 1] elif (arg == "-m"): metric = argv[i + 1] else: Util.error("Flag '" + argv[i] + "' not recognized") i = i + 1 else: ifiles.append(argv[i]) i = i + 1 if (version): print "Version: " + Version.__version__ return # Deal with legend entries if (leg is not None): leg = leg.split(',') for i in range(0, len(leg)): leg[i] = leg[i].replace('_', ' ') if (latlonRange is not None and len(latlonRange) != 4): Util.error("-llRange <values> must have exactly 4 values") if (len(ifiles) > 0): data = Data.Data(ifiles, clim=climFile, climType=climType, dates=dates, offsets=offsets, locations=locations, latlonRange=latlonRange, training=training) else: data = None if (listThresholds or listQuantiles or listLocations or listDates): if (len(ifiles) == 0): Util.error( "Files are required in order to list thresholds or quantiles") if (listThresholds): print "Thresholds:", for threshold in data.getThresholds(): print "%g" % threshold, print "" if (listQuantiles): print "Quantiles:", for quantile in data.getQuantiles(): print "%g" % quantile, print "" if (listLocations): print " id lat lon elev" for station in data.getStations(): print "%6d %7.2f %7.2f %7.1f" % (station.id(), station.lat(), station.lon(), station.elev()) print "" if (listDates): dates = data.getAxisValues("date") dates = Util.convertToYYYYMMDD(dates) for date in dates: print "%d" % date print "" return elif (len(argv) == 1 or len(ifiles) == 0 or metric is None): showDescription(data) return if (figSize is not None): figSize = figSize.split(',') if (len(figSize) != 2): print "-fs figSize must be in the form: width,height" sys.exit(1) m = None # Handle special plots if (doHist): pl = Output.Hist(metric) elif (doSort): pl = Output.Sort(metric) elif (metric == "pithist"): m = Metric.Pit("pit") pl = Output.PitHist(m) elif (metric == "obsfcst"): pl = Output.ObsFcst() elif (metric == "timeseries"): pl = Output.TimeSeries() elif (metric == "meteo"): pl = Output.Meteo() elif (metric == "qq"): pl = Output.QQ() elif (metric == "cond"): pl = Output.Cond() elif (metric == "against"): pl = Output.Against() elif (metric == "count"): pl = Output.Count() elif (metric == "scatter"): pl = Output.Scatter() elif (metric == "change"): pl = Output.Change() elif (metric == "spreadskill"): pl = Output.SpreadSkill() elif (metric == "taylor"): pl = Output.Taylor() elif (metric == "error"): pl = Output.Error() elif (metric == "freq"): pl = Output.Freq() elif (metric == "roc"): pl = Output.Roc() elif (metric == "droc"): pl = Output.DRoc() elif (metric == "droc0"): pl = Output.DRoc0() elif (metric == "drocnorm"): pl = Output.DRocNorm() elif (metric == "reliability"): pl = Output.Reliability() elif (metric == "invreliability"): pl = Output.InvReliability() elif (metric == "igncontrib"): pl = Output.IgnContrib() elif (metric == "economicvalue"): pl = Output.EconomicValue() elif (metric == "marginal"): pl = Output.Marginal() else: # Standard plots # Attempt at automating metrics = Metric.getAllMetrics() m = None for mm in metrics: if (metric == mm[0].lower() and mm[1].isValid()): m = mm[1]() break if (m is None): m = Metric.Default(metric) m.setAggregator(cType) # Output type if (type == "plot" or type == "text" or type == "map" or type == "maprank"): pl = Output.Default(m) pl.setShowAcc(doAcc) else: Util.error("Type not understood") # Rest dimension of '-x' is not allowed if (xdim is not None and not pl.supportsX()): Util.warning(metric + " does not support -x. Ignoring it.") xdim = None # Reset dimension if 'threshold' is not allowed if (xdim == "threshold" and ((not pl.supportsThreshold()) or (not m.supportsThreshold()))): Util.warning(metric + " does not support '-x threshold'. Ignoring it.") thresholds = None xdim = None # Create thresholds if needed if ((thresholds is None) and (pl.requiresThresholds() or (m is not None and m.requiresThresholds()))): data.setAxis("none") obs = data.getScores("obs")[0] fcst = data.getScores("fcst")[0] smin = min(min(obs), min(fcst)) smax = max(max(obs), max(fcst)) thresholds = np.linspace(smin, smax, 10) Util.warning("Missing '-r <thresholds>'. Automatically setting\ thresholds.") # Set plot parameters if (markerSize is not None): pl.setMarkerSize(markerSize) if (lineWidth is not None): pl.setLineWidth(lineWidth) if (labFontSize is not None): pl.setLabFontSize(labFontSize) if (legFontSize is not None): pl.setLegFontSize(legFontSize) if (tickFontSize is not None): pl.setTickFontSize(tickFontSize) if (XRotation is not None): pl.setXRotation(XRotation) if (MajorLength is not None): pl.setMajorLength(MajorLength) if (MinorLength is not None): pl.setMinorLength(MinorLength) if (MajorWidth is not None): pl.setMajorWidth(MajorWidth) if (Bottom is not None): pl.setBottom(Bottom) if (Top is not None): pl.setTop(Top) if (Right is not None): pl.setRight(Right) if (Left is not None): pl.setLeft(Left) if (Pad is not None): pl.setPad(None) if (binType is not None): pl.setBinType(binType) if (showPerfect is not None): pl.setShowPerfect(showPerfect) if (xlim is not None): pl.setXLim(xlim) if (ylim is not None): pl.setYLim(ylim) if (clim is not None): pl.setCLim(clim) pl.setFilename(ofile) pl.setThresholds(thresholds) pl.setLegend(leg) pl.setFigsize(figSize) pl.setDpi(dpi) pl.setAxis(xdim) pl.setShowMargin(not noMargin) pl.setYlabel(ylabel) pl.setXlabel(xlabel) pl.setTitle(title) if (type == "text"): pl.text(data) elif (type == "map"): pl.map(data) elif (type == "maprank"): pl.setShowRank(True) pl.map(data) else: pl.plot(data)
def __init__(self, filename): import csv Input.__init__(self, filename) file = open(filename, 'r') self._units = "Unknown units" self._variable = "Unknown" self._pit = None self._dates = set() self._offsets = set() self._stations = set() self._quantiles = set() self._thresholds = set() fields = dict() obs = dict() fcst = dict() cdf = dict() pit = dict() x = dict() indices = dict() header = None # Default values if columns not available offset = 0 date = 0 lat = 0 lon = 0 elev = 0 import time start = time.time() # Read the data into dictionary with (date,offset,lat,lon,elev) as key and obs/fcst as values for rowstr in file: if (rowstr[0] == "#"): curr = rowstr[1:] curr = curr.split() if (curr[0] == "variable:"): self._variable = curr[1] elif (curr[0] == "units:"): self._units = curr[1] else: Util.warning("Ignoring line '" + rowstr.strip() + "' in file '" + filename + "'") else: row = rowstr.split() if (header is None): # Parse the header so we know what each column represents header = row for i in range(0, len(header)): att = header[i] if (att == "date"): indices["date"] = i elif (att == "offset"): indices["offset"] = i elif (att == "lat"): indices["lat"] = i elif (att == "lon"): indices["lon"] = i elif (att == "elev"): indices["elev"] = i elif (att == "obs"): indices["obs"] = i elif (att == "fcst"): indices["fcst"] = i else: indices[att] = i # Ensure we have required columns requiredColumns = ["obs", "fcst"] for col in requiredColumns: if (col not in indices): msg = "Could not parse %s: Missing column '%s'" % ( filename, col) Util.error(msg) else: if (len(row) is not len(header)): Util.error( "Incorrect number of columns (expecting %d) in row '%s'" % (len(header), rowstr.strip())) if ("date" in indices): date = self._clean(row[indices["date"]]) self._dates.add(date) if ("offset" in indices): offset = self._clean(row[indices["offset"]]) self._offsets.add(offset) if ("id" in indices): id = self._clean(row[indices["id"]]) else: id = np.nan if ("lat" in indices): lat = self._clean(row[indices["lat"]]) if ("lon" in indices): lon = self._clean(row[indices["lon"]]) if ("elev" in indices): elev = self._clean(row[indices["elev"]]) station = Station.Station(id, lat, lon, elev) self._stations.add(station) key = (date, offset, lat, lon, elev) obs[key] = self._clean(row[indices["obs"]]) fcst[key] = self._clean(row[indices["fcst"]]) quantileFields = self._getQuantileFields(header) thresholdFields = self._getThresholdFields(header) if "pit" in indices: pit[key] = self._clean(row[indices["pit"]]) for field in quantileFields: quantile = float(field[1:]) self._quantiles.add(quantile) key = (date, offset, lat, lon, elev, quantile) x[key] = self._clean(row[indices[field]]) for field in thresholdFields: threshold = float(field[1:]) self._thresholds.add(threshold) key = (date, offset, lat, lon, elev, threshold) cdf[key] = self._clean(row[indices[field]]) end = time.time() file.close() self._dates = list(self._dates) self._offsets = list(self._offsets) self._stations = list(self._stations) self._quantiles = list(self._quantiles) self._thresholds = np.array(list(self._thresholds)) Ndates = len(self._dates) Noffsets = len(self._offsets) Nlocations = len(self._stations) Nquantiles = len(self._quantiles) Nthresholds = len(self._thresholds) # Put the dictionary data into a regular 3D array self._obs = np.zeros([Ndates, Noffsets, Nlocations], 'float') * np.nan self._fcst = np.zeros([Ndates, Noffsets, Nlocations], 'float') * np.nan if (len(pit) != 0): self._pit = np.zeros([Ndates, Noffsets, Nlocations], 'float') * np.nan self._cdf = np.zeros([Ndates, Noffsets, Nlocations, Nthresholds], 'float') * np.nan self._x = np.zeros([Ndates, Noffsets, Nlocations, Nquantiles], 'float') * np.nan for d in range(0, len(self._dates)): date = self._dates[d] end = time.time() for o in range(0, len(self._offsets)): offset = self._offsets[o] for s in range(0, len(self._stations)): station = self._stations[s] lat = station.lat() lon = station.lon() elev = station.elev() key = (date, offset, lat, lon, elev) if (key in obs): self._obs[d][o][s] = obs[key] if (key in fcst): self._fcst[d][o][s] = fcst[key] if (key in pit): self._pit[d][o][s] = pit[key] for q in range(0, len(self._quantiles)): quantile = self._quantiles[q] key = (date, offset, lat, lon, elev, quantile) if (key in x): self._x[d, o, s, q] = x[key] for t in range(0, len(self._thresholds)): threshold = self._thresholds[t] key = (date, offset, lat, lon, elev, threshold) if (key in cdf): self._cdf[d, o, s, t] = cdf[key] end = time.time() maxStationId = np.nan for station in self._stations: if (np.isnan(maxStationId)): maxStationId = station.id() elif (station.id() > maxStationId): maxStationId = station.id() counter = 0 if (not np.isnan(maxStationId)): counter = maxStationId + 1 for station in self._stations: if (np.isnan(station.id())): station.id(counter) counter = counter + 1 self._dates = np.array(self._dates) self._offsets = np.array(self._offsets)