def thisMonthOverTime(month): for fname in ['MEAN_TEMP']: findex = daily.fields._fields.index(fname) data = [] for year in range(min(cityData.keys()),2010): avg = filterAverage(YearMonthFilter(year, month), findex) if avg != None: data.append((year,avg.average)) lineFit = linear.linearTrend(data) plot = gnuplot.Plot("ThisMonthOverTime_%s_%s" % (fname, monthName(month)), yaxis=2) plot.open() plot.addLine(gnuplot.Line(fname, data, lineColour='blue')) plot.addLine(gnuplot.Line('linear', lineFit, lineColour='green')) plot.plot() plot.close() for fname in ['TOTAL_SNOW_CM', 'TOTAL_PRECIP_MM']: findex = daily.fields._fields.index(fname) data = [] for year in range(min(cityData.keys()),2010): data.append((year, filterSum(YearMonthFilter(year, month), findex).total)) lineFit = linear.linearTrend(data) plot = gnuplot.Plot("ThisMonthOverTime_%s_%s" % (fname, monthName(month)), yaxis=2) plot.open() plot.addLine(gnuplot.Line(fname, data, lineColour='blue')) plot.addLine(gnuplot.Line('linear', lineFit, lineColour='green')) plot.plot() plot.close()
def annualOrderedByMonth(monthFilter, field, cumulative): valByYears = yearDatas(monthFilter, field, now.year, cumulative) plotDataOtherYears = [] plotDataThisYear = [] plotTics = [] plotIndex = 0 divider = 1 ylabel = field.units if max(valByYears.values()) > 10000: divider = 1000.0 ylabel = '1000s of %s' % field.units valByYears = makeFit(valByYears, 75) plotIndex = 0 last30Years = [] yearsByVal = reverseDictionary(valByYears) for val in sorted(yearsByVal.keys()): yearsWithThisVal = yearsByVal[val] for plotYear in yearsWithThisVal: if now.year == plotYear: plotDataThisYear.append((plotIndex, val/divider)) else: plotDataOtherYears.append((plotIndex, val/divider)) plotTics.append('"%s" %d' % (plotYear, plotIndex)) plotIndex += 1 if (plotYear > now.year - 31) and (plotYear < now.year): last30Years.append(val/divider) avg = sum(last30Years) / len(last30Years) std = numpy.std([float(a) for a in last30Years]) #print if len(yearsByVal.keys()) > 0: plot = gnuplot.Plot('%s/svg/yearOrdering.%s.%s' % (cityName, monthFilter.filenamePart, field.title), yaxis=2) legend='on left' if max(yearsByVal.keys()) < 0: legend='bottom right' ymin = None if cumulative: ymin = 0 plot.open(xtics=plotTics, xticsRotate=90, xticsFont='Arial,10', legend=legend, ymin=ymin, title='%s %s for %s' % (cityName.capitalize(), field.title.lower(), monthFilter.chartTitle), margins=[6,8,2,3], ylabel=ylabel ) plot.addLine(gnuplot.Line('30-year std-dev', ((-1, avg-std), (plotIndex, avg-std), (plotIndex, avg+std), (-1, avg+std)), lineColour='#e3e3e3', plot='filledcurves')) plot.addLine(gnuplot.Line('Other years', plotDataOtherYears, plot='boxes')) plot.addLine(gnuplot.Line('This year', plotDataThisYear, plot='boxes')) plot.addLine(gnuplot.Line('30-Year Average', ((0, avg), (plotIndex-1, avg)))) plot.plot() plot.close()
def plotline(valueByYear, filename, chartTitle, yaxisLabel): data = sorted(tuple(valueByYear.items())) if len(data) > 1: lineFit = linear.linearTrend(tuple((a[0], float(a[1])) for a in data)) plot = gnuplot.Plot(filename, yaxis=2) plot.open(title=chartTitle, ylabel=yaxisLabel) if len(data) > 1: plot.addLine( gnuplot.Line("Linear", lineFit, lineColour='green', plot='lines')) plot.addLine(gnuplot.Line("Temp", data, lineColour='purple')) plot.plot() plot.close() return plot.fname
def plotMonthly(name, filterFunc): sunnyHoursByMonth = totalHoursByMonth(filterFunc) # monthVals = {} # currentYearVals = [] lastYearVals = [] # for year in sorted(sunnyHoursByMonth.keys()): for month in sorted(sunnyHoursByMonth[year].keys()): if month not in monthVals: monthVals[month] = [] sunnySum = sunnyHoursByMonth[year][month] average = sunnySum.average() if year != now.year: monthVals[month].append(average) if year == now.year: print "%d/%02d: %.2f" % (year, month, average) currentYearVals.append((month, average)) elif year == now.year - 1: lastYearVals.append((month, average)) # plotAvg = [] plotMin = [] plotMax = [] # for month in sorted(monthVals.keys()): Avg = sum(monthVals[month]) / len(monthVals[month]) Min = min(monthVals[month]) Max = max(monthVals[month]) print "%d: avg=%.2f, min=%.2f, max=%.2f" % (month, Avg, Min, Max) plotAvg.append((month, Avg)) plotMin.append((month, Min)) plotMax.append((month, Max)) # # plot = gnuplot.Plot('ottawa/%s.%s' % (now.year, name), yaxis=2) plot.open(xtics=[ '"Jan" 1', '"Feb" 2', '"Mar" 3', '"Apr" 4', '"May" 5', '"Jun" 6', '"Jul" 7', '"Aug" 8', '"Sep" 9', '"Oct" 10', '"Nov" 11', '"Dec" 12' ]) plot.addLine( gnuplot.Line('%d' % now.year, currentYearVals, lineColour='purple')) plot.addLine( gnuplot.Line('%d' % (now.year - 1), lastYearVals, lineColour='orange')) plot.addLine(gnuplot.Line('Average', plotAvg, lineColour='green')) plot.addLine(gnuplot.Line('Min', plotMin, lineColour='blue')) plot.addLine(gnuplot.Line('Max', plotMax, lineColour='red')) plot.plot() plot.close()
def plotDifference(city1Name, city2Name, field, cumulative): city1ByYear = getValuesByYear(city1Name, field, cumulative) city2ByYear = getValuesByYear(city2Name, field, cumulative) # bothYears = sorted( list(sets.Set(city1ByYear.keys()) & sets.Set(city2ByYear.keys()))) # plotData = [] for year in bothYears: diff = city2ByYear[year] - city1ByYear[year] print "%d: %.1f" % (year, diff) plotData.append((year, diff)) # plot = gnuplot.Plot("%s_minus_%s_Annual_%s" % ('farm', 'ottawa', daily.fields[field].name), yaxis=2) plot.open() plot.addLine( gnuplot.Line("Temp", plotData, lineColour='purple', plot='boxes')) plot.plot() plot.close()
# thisyearsnowy = totalHours(weather.hourly.WEATHER, Snowy(year)) snowData.append((year, thisyearsnowy)) print "Snow: %d\t%d" % (year, thisyearsnowy) print sunRecord sunLineFit = {10: [], 15: [], 20: [], 25: [], 30: []} for t in sunData: sunLineFit[t] = linear.linearTrend(sunData[t]) snowlineFit = linear.linearTrend(snowData) plot = gnuplot.Plot("ottawa/SunnyHours", yaxis=2) plot.open(ymin=0) for t in sunData: plot.addLine(gnuplot.Line("Trend %d" % t, sunLineFit[t], plot='lines')) plot.addLine(gnuplot.Line("Sun %d" % t, sunData[t])) plot.plot() plot.close() plot = gnuplot.Plot("ottawa/SnowHours", yaxis=2) plot.open(ymin=0) plot.addLine( gnuplot.Line("Trend", snowlineFit, lineColour='green', plot='lines')) plot.addLine(gnuplot.Line("Snow", snowData, lineColour='purple')) plot.plot() plot.close()
def count(cityName, data, expr, name, verbose=True, skipIncomplete=False, lineChart=False): countByYear = {} thisYearCount = 0 for baseyear in range(data.minYear, data.maxYear - YEAR_CROSS() + 1): countByYear[baseyear] = None countThisYear = 0 starttime = datetime.datetime(baseyear, START_MONTH, START_DAY) endtime = datetime.datetime(baseyear + YEAR_CROSS(), END_MONTH, END_DAY) hourdiff = (endtime - starttime).days * 24 hoursWithObservations = 0 thisRunLen = 0 #print baseyear for day in hourly.hourRange(starttime, endtime): dayValues = data.get(day, None) val = None flag = '' if dayValues != None: #print day, dayValues val, flag = expr(dayValues) if val != None: hoursWithObservations += 1 if val and not (skipIncomplete and 'I' in flag): countThisYear += 1 if baseyear + YEAR_CROSS() == now.year: if verbose: print "Count: %s" % day, float(val), flag if (day.date() == min( endtime.date() - datetime.timedelta(1), datetime.date.today())): # We only care about this year's count if the final day of # this year actually met the criteria thisYearCount = countThisYear else: #if baseyear == 2014: # print "Skip: %s" % day thisRunLen = 0 if (hoursWithObservations >= hourdiff * 85 / 100): # or baseyear + YEAR_CROSS() == now.year: countByYear[baseyear] = countThisYear if verbose: print baseyear, countThisYear, hoursWithObservations else: # skip years with fewer than 90% of days recorded if verbose: print("Skipping %d because it has only %u/%u records" % (baseyear, hoursWithObservations, hourdiff)) #del countByYear[now.year] (fewest, most) = keyOfMinMaxValue(countByYear) if verbose: print "fewest %s was %s (%d)" % (name, fewest, countByYear[fewest[0]]) print "most %s was %s (%d)" % (name, most, countByYear[most[0]]) eThisYear = now.year if YEAR_CROSS(): eThisYear -= 1 avgKeys = filter(lambda t: t in countByYear, range(eThisYear - 30, eThisYear)) counts = [countByYear[a] for a in avgKeys] counts = filter(lambda t: t != None, counts) avg = float(sum(counts)) / len(counts) if verbose: print "average %s is %f" % (name, avg) #print thisYearCount mostSince = None prevRecordYears = [] prevRecord = 0 if eThisYear in countByYear: #thisYearCount = countByYear[eThisYear] if thisYearCount > 0: for year in reversed(sorted(countByYear.keys())): if mostSince == None and year != eThisYear and countByYear[ year] >= thisYearCount: mostSince = year if year != eThisYear: if countByYear[year] > prevRecord: prevRecord = countByYear[year] prevRecordYears = [year] elif countByYear[year] == prevRecord: prevRecordYears.append(year) if verbose: print now.year, name, "was", countByYear[eThisYear] if mostSince != None: print "Most since %d." % (mostSince) print( "%d of past %d years had more %s" % (len(filter(lambda t: countByYear[t] > thisYearCount, avgKeys)), len(avgKeys), name), filter(lambda t: countByYear[t] > thisYearCount, avgKeys)) print( "%d of past %d years had equal %s" % (len(filter(lambda t: countByYear[t] == thisYearCount, avgKeys)), len(avgKeys), name), filter(lambda t: countByYear[t] == thisYearCount, avgKeys)) print( "%d of past %d years had fewer %s" % (len(filter(lambda t: countByYear[t] < thisYearCount, avgKeys)), len(avgKeys), name), filter(lambda t: countByYear[t] < thisYearCount, avgKeys)) print if lineChart: plotdata = [] for (year, plotCount) in countByYear.iteritems(): if plotCount != None: plotdata.append((year, plotCount)) lineFit = linear.linearTrend(plotdata) plot = gnuplot.Plot( "%s/svg/%s_count_%s-%u_before_%s" % (cityName, name.replace(' ', '_'), monthName(START_MONTH), START_DAY, lastDay().replace(' ', '-')), yaxis=2) plot.open(title='%s number of %s between %s %u and %s' % (cityName.capitalize(), name, monthName(START_MONTH), START_DAY, lastDay())) plot.addLine( gnuplot.Line("Linear", lineFit, lineColour='green', plot='lines')) plot.addLine(gnuplot.Line("Count", plotdata, lineColour='purple')) plot.plot() plot.close() else: barPlotData = makeFit(countByYear, 75) filename = ("%s/svg/%s_count_%s-%u_to_%s_bar" % (cityName, name.replace(' ', '_'), monthName(START_MONTH), START_DAY, lastDay().replace(' ', '-'))) plotdict(barPlotData, filename=filename, chartTitle=('%s number of %s between %s %u and %s' % (cityName.capitalize(), name, monthName(START_MONTH), START_DAY, lastDay())), yaxisLabel='', thisYear=eThisYear, ymin=0) return (filename, thisYearCount, mostSince, prevRecord, prevRecordYears)
def createPlot(plotData, city, name, yaxisLabel, fieldEnglishName, units, plotDate, plotZeros, verbose): yearByValue = {} valueByYear = {} for year in plotData: # value = plotData[year] #if value == None: # value = 0 if (value not in yearByValue): yearByValue[value] = [year] else: yearByValue[value].append(year) valueByYear[year] = value #print year, value top10Years = valuesOfHighest10Keys(yearByValue) bottom10Years = valuesOfLowest10Keys(yearByValue) allfname = plotdict( valueByYear, filename='{city}/svg/{name}.all'.format(**locals()), chartTitle='%s daily %s for %s' % (city.title(), fieldEnglishName, plotDate.strftime('%b %d')), yaxisLabel=yaxisLabel, thisYear=plotDate.year, plotZeros=True, showAverage=False, ) lineFname = plotline( valueByYear, filename='{city}/svg/{name}.line'.format(**locals()), chartTitle='%s daily %s for %s' % (city.title(), fieldEnglishName, plotDate.strftime('%b %d')), yaxisLabel=yaxisLabel, ) top10Fname = plotdict( filterDict(valueByYear, top10Years), filename='%s/svg/%s.top10' % (city, name), chartTitle='Top 10 %s daily %ss for %s' % (city.title(), fieldEnglishName, plotDate.strftime('%b %d')), yaxisLabel=yaxisLabel, thisYear=plotDate.year, plotZeros=True, showAverage=False, ) print(filterDict(valueByYear, bottom10Years)) bottom10Fname = plotdict( filterDict(valueByYear, bottom10Years), filename='%s/svg/%s.bottom10' % (city, name), chartTitle='Bottom 10 %s daily %ss for %s' % (city.title(), fieldEnglishName, plotDate.strftime('%b %d')), yaxisLabel=yaxisLabel, thisYear=plotDate.year, plotZeros=True, showAverage=False, ) values = sorted(yearByValue.keys()) # chartTicks = [] chartDataOtherYears = [] chartDataThisYear = [] chartIndex = 0 count = 0 recentValues = [] maxValue = max(values) for value in values: # number of years with this value thiscount = len(yearByValue[value]) # recentYearsWithThisValue = tuple( filter(lambda y: y < plotDate.year and y > plotDate.year - 31, yearByValue[value])) if verbose: lrecent = len(recentValues) lrecentVal = len(recentYearsWithThisValue) isMedian = (lrecent <= 15) and (lrecent + lrecentVal > 15) #print lrecent, lrecentVal, map(float, recentValues) print("{}-{}) {:.1f}{}, {}{}".format(count + 1, len(valueByYear) - count, value, units, yearByValue[value], ["", "*** median"][isMedian])) recentValues += [value] * len(recentYearsWithThisValue) count += thiscount # if plotZeros == False and value == 0: # We've been told to skip zeros, so we don't plot them continue for year in yearByValue[value]: if year == plotDate.year: chartDataThisYear.append((chartIndex, value, '# %d' % year)) else: chartDataOtherYears.append((chartIndex, value, '# %d' % year)) chartTicks.append('"%d" %d' % (year, chartIndex)) chartIndex += 1 legend = 'on left' if maxValue < 0: legend = 'on right bottom' bmargin = 5 plot = gnuplot.Plot('%s/svg/%s.yearOrdering' % (city, name), yaxis=2, output='svg') # plot.open( title='%s daily %s for %s' % (city.title(), fieldEnglishName, plotDate.strftime('%b %d')), xtics=chartTicks, xticsRotate=90, xticsFont='Arial,10', legend=legend, margins=[6, 8, 2, bmargin], ylabel=yaxisLabel, # ymin=0, xmin=-1, xmax=chartIndex) plot.addLine( gnuplot.Line('Other years', chartDataOtherYears, plot='boxes', lineColour='0x6495ED')) plot.addLine( gnuplot.Line('This year', chartDataThisYear, plot='boxes', lineColour='0x556B2F')) plot.plot() plot.close() return lineFname, bottom10Fname, top10Fname
def annualPlots(city, year): for field in [ Avg(daily.MAX_TEMP, daily.MIN_TEMP, "Temperature"), FractionVal(daily.MAX_TEMP, "Max temp"), FractionVal(daily.MIN_TEMP, "Min temp"), FractionVal(daily.MEAN_TEMP, "Mean temp"), FractionVal(daily.MEAN_HUMIDITY, "Mean humidity"), FractionVal(daily.SNOW_ON_GRND_CM, "Snow-on-the-ground"), FractionVal(daily.AVG_WIND, "Wind"), FractionVal(daily.AVG_DEWPOINT, "Mean dewpoint"), ]: print(field.name) current = [] average = [] lowest = [] highest = [] stdLow = [] stdHigh = [] for month in range(1, 13): thisyear = filterAverage(YearMonthFilter(year, month), field) normal = normalMonthlyAverage( year, YearMonthFilter(year=None, month=month), field) average.append("%d %.1f" % (month, normal.average)) stdLow.append("%d %.1f" % (month, normal.average - normal.std)) stdHigh.append("%d %.1f" % (month, normal.average + normal.std)) lowest.append("%d %.1f" % (month, normal.minimum)) highest.append("%d %.1f" % (month, normal.maximum)) if thisyear != None and thisyear.count >= 15: current.append("%d %.1f" % (month, thisyear.average)) units = field.units print( "%10s-%10s: %+6.2f%s (%d: %+6.2f%s, normal: %+6.2f%s, extremeMin: %+6.2f%s %s, extremeMax: %+6.2f%s %s)" % (monthName(month), field.name, (thisyear.average - normal.average), units, year, thisyear.average, units, normal.average, units, normal.minimum, units, normal.minYear, normal.maximum, units, normal.maxYear)) stdData = stdLow + list(reversed(stdHigh)) plot = gnuplot.Plot( "%s/svg/%d_%s" % (city, year, field.name.replace(' ', '_').replace('℃', 'C')), yaxis=2) plot.open(title="%s %s" % (city, field.title), xtics=[ '"Jan" 1', '"Feb" 2', '"Mar" 3', '"Apr" 4', '"May" 5', '"Jun" 6', '"Jul" 7', '"Aug" 8', '"Sep" 9', '"Oct" 10', '"Nov" 11', '"Dec" 12' ]) plot.addLine( gnuplot.Line('30-year std-dev', stdData, lineColour='#e3e3e3', plot='filledcurves')) plot.addLine( gnuplot.Line("30-year normal", average, lineColour='#007700')) plot.addLine(gnuplot.Line("Record min", lowest, lineColour='blue')) plot.addLine(gnuplot.Line("Record max", highest, lineColour='red')) plot.addLine(gnuplot.Line("%d" % year, current, lineColour='purple')) plot.plot() plot.close() for field in (FractionVal(daily.TOTAL_SNOW_CM, "Snowfall"), FractionVal(daily.TOTAL_RAIN_MM, "Rainfall"), FractionVal(daily.TOTAL_PRECIP_MM, "Precip")): print(field.name) print("%s\t%s\t%s\t%s\t%s" % ("Month", "%d" % year, "Average", "Record max", "Record min")) current = [] average = [] lowest = [] highest = [] stdLow = [] stdHigh = [] for month in range(1, 13): thisyear = filterSum(YearMonthFilter(year, month), field) normal = normalMonthlySum(year, YearMonthFilter(year, month), field) if normal != None: average.append((month, normal.average)) stdLow.append("%d %.1f" % (month, normal.average - normal.std)) stdHigh.append("%d %.1f" % (month, normal.average + normal.std)) lowest.append((month, normal.minimum)) highest.append((month, normal.maximum)) if thisyear.count > 13: current.append((month, thisyear.total)) units = field.units print( "%10s-%10s: %+6.1f%s (%d: %+6.1f%s, normal: %+6.1f%s, extremeMin: %+6.1f%s %s, extremeMax: %+6.1f%s %s)" % (monthName(month), field.name, (thisyear.total - normal.average), units, year, thisyear.total, units, normal.average, units, normal.minimum, units, normal.minYear, normal.maximum, units, normal.maxYear)) if (len(current) > 0 and len(average) > 0 and len(lowest) > 0 and len(highest) > 0): stdData = stdLow + list(reversed(stdHigh)) plot = gnuplot.Plot( "%s/svg/%d_%s" % (city, year, field.name.replace(' ', '_').replace('℃', 'C')), yaxis=2) plot.open(title="%s %s" % (city, field.title), xtics=[ '"Jan" 1', '"Feb" 2', '"Mar" 3', '"Apr" 4', '"May" 5', '"Jun" 6', '"Jul" 7', '"Aug" 8', '"Sep" 9', '"Oct" 10', '"Nov" 11', '"Dec" 12' ]) plot.addLine( gnuplot.Line('30-year std-dev', stdData, lineColour='#e3e3e3', plot='filledcurves')) plot.addLine( gnuplot.Line("30-year normal", average, lineColour='#007700')) plot.addLine(gnuplot.Line("Record min", lowest, lineColour='blue')) plot.addLine(gnuplot.Line("Record max", highest, lineColour='red')) plot.addLine( gnuplot.Line("%d" % year, current, lineColour='purple')) plot.plot() plot.close()
try: lineFit = linear.linearTrend(data) except ZeroDivisionError: print(data) raise for i in range(len(data)): year, val = data[i] lfitv = lineFit[i][1] print(year, float(val), lfitv) plot = gnuplot.Plot("%s/svg/Annual_%s" % (city, fname), yaxis=2) plot.open(title='%s %s per year' % (city.capitalize(), field.title), ylabel='%s in %s' % (field.title, field.units)) plot.addLine( gnuplot.Line("Linear", lineFit, lineColour='green', plot='lines')) plot.addLine(gnuplot.Line("Temp", data, lineColour='purple')) plot.plot() plot.close() for field in [ FractionVal(daily.TOTAL_SNOW_CM, 'snow'), FractionVal(daily.SNOW_ON_GRND_CM, 'snowpack'), FractionVal(daily.TOTAL_RAIN_MM, 'rain'), FractionVal(daily.TOTAL_PRECIP_MM, 'precipitation') ]: fname = field.name print(fname) data = getAnnualValues(cityData, field, True) for year, val in data: print(year, float(val))
progressiveError.append(abs(progressivePrediction - val)) lineFitError.append(abs(lfitv - val)) sameError.append(abs(float(data[i - 1][1] - val))) co2FitError.append(abs(co2fv - val)) print year, float(val), lfitv co2ErrorBarSize = sorted(co2FitError)[len(co2FitError) * 95 / 100] co2ErrorBarPlot = [] for year, val in co2Fit: co2ErrorBarPlot.append((year, val, co2ErrorBarSize)) plot = gnuplot.Plot("%s/Annual_Prediction_%s" % (city, fname), yaxis=2) plot.open(title='%s %s per year' % (city.capitalize(), field.title), ylabel='%s in %s' % (field.title, field.units)) plot.addLine( gnuplot.Line("Linear", lineFit, lineColour='green', plot='lines')) plot.addLine( gnuplot.Line("Extrapolation", progressiveLineFit, lineColour='red', plot='lines')) plot.addLine(gnuplot.Line("Co2", co2Fit, lineColour='blue')) plot.addLine( gnuplot.Line("Co2 Error", co2ErrorBarPlot, lineColour='blue', plot='errorbars')) plot.addLine(gnuplot.Line("Temp", data, lineColour='purple')) plot.plot() plot.close()
def plotdict(valueByYear, filename, chartTitle, yaxisLabel, thisYear, plotZeros=True, output='svg', ymin=None, showAverage=True): yearByValue = reverseDict(valueByYear) values = sorted(yearByValue.keys()) # chartTicks = [] chartDataOtherYears = [] chartDataThisYear = [] chartIndex = 0 maxValue = max(values) if showAverage: recentYears = tuple( filter(lambda t: t in valueByYear, range(thisYear - 30, thisYear))) recentValues = [valueByYear[t] for t in recentYears] recentAvg = sum(recentValues) / len(recentValues) recentStd = numpy.std(tuple(map(float, recentValues))) longestXTick = 0 for value in values: # number of years with this value #print 'val="%s"' % value thiscount = len(yearByValue[value]) # if plotZeros == False and value == 0: # We've been told to skip zeros, so we don't plot them continue for year in yearByValue[value]: if year == thisYear: chartDataThisYear.append((chartIndex, value, '# ' + str(year))) else: chartDataOtherYears.append( (chartIndex, value, '# ' + str(year))) xtick = str(year) longestXTick = max(longestXTick, len(xtick)) chartTicks.append('"{date}" {index}'.format(date=xtick, index=chartIndex)) chartIndex += 1 legend = 'on left' if maxValue < 0: legend = 'on right bottom' bmargin = 2 + longestXTick // 2 plot = gnuplot.Plot(filename, yaxis=2, output=output) # #, xticsFont='Arial,10' plot.open(title=chartTitle, xtics=chartTicks, xticsRotate=90, xticsFont='Arial,20', legend=legend, margins=[6, 8, 2, bmargin], ylabel=yaxisLabel, ymin=ymin, xmin=-1, xmax=chartIndex) if showAverage: plot.addLine( gnuplot.Line('30-year 2*std-dev', ((-1, recentAvg - recentStd * 2), (chartIndex, recentAvg - recentStd * 2), (chartIndex, recentAvg + recentStd * 2), (-1, recentAvg + recentStd * 2)), lineColour='#f1f1f1', plot='filledcurves')) plot.addLine( gnuplot.Line('30-year std-dev', ((-1, recentAvg - recentStd), (chartIndex, recentAvg - recentStd), (chartIndex, recentAvg + recentStd), (-1, recentAvg + recentStd)), lineColour='#e3e3e3', plot='filledcurves')) plot.addLine( gnuplot.Line('30-year average', ((-1, recentAvg), (chartIndex, recentAvg)), lineColour='0x000000')) plot.addLine( gnuplot.Line('Other years', chartDataOtherYears, plot='boxes', lineColour='0x6495ED')) plot.addLine( gnuplot.Line('This year', chartDataThisYear, plot='boxes', lineColour='0x556B2F')) plot.plot() plot.close() return plot.fname
def first(cityName, expr, name, dateRange, excludeThisYear=False, order="first", yaxisLabel=None, run=1, limitToMostRecentYears=None, verboseIfInDateRange=None, ): data = daily.load(cityName) if limitToMostRecentYears != None: ndata = daily.Data() yearsAgo = datetime.date(now.year-limitToMostRecentYears, now.month, now.day) for key in data: if key >= yearsAgo: ndata[key] = data[key] data = ndata fieldDict = { 'min': Value(daily.MIN_TEMP), 'max': Value(daily.MAX_TEMP), 'tempSpan': ValueDiff(daily.MAX_TEMP, daily.MIN_TEMP), 'rain': Value(daily.TOTAL_RAIN_MM), 'humidex': Value(daily.MAX_HUMIDEX), 'snow': Value(daily.TOTAL_SNOW_CM), 'snowpack': ValueEmptyZero(daily.SNOW_ON_GRND_CM), 'windgust': Value(daily.SPD_OF_MAX_GUST_KPH), 'wind': Value(daily.AVG_WIND), 'windchill': Value(daily.MIN_WINDCHILL), 'avgWindchill': Value(daily.MIN_WINDCHILL), } fieldValues = fieldDict.values() referencedValues=[] for fieldName in fieldDict.keys(): if '{'+fieldName+'}' in expr: referencedValues.append(fieldName) firstByYear = {} eThisYear = now.year endTimeThisYear = datetime.date(now.year, dateRange.endMonth, dateRange.endDay) if dateRange.yearCross() and now < endTimeThisYear: eThisYear -= 1 for baseyear in range(data.minYear, data.maxYear-dateRange.todayIsYearCross()+1): try: dayOffset = datacache.readCache(cityName, baseyear, '{}.{}.{}.{}'.format(name,order,str(dateRange),expr)) if dayOffset != None: firstByYear[baseyear] = dayOffset continue except datacache.NotInCache: pass starttime = datetime.date(baseyear, dateRange.startMonth, dateRange.startDay) endtime = datetime.date(baseyear+dateRange.yearCross(), dateRange.endMonth, dateRange.endDay) #print baseyear, starttime, endtime dayRange = daily.dayRange(starttime,endtime) if order=="last": dayRange = daily.dayRange(endtime-datetime.timedelta(1), starttime-datetime.timedelta(1), -1) if excludeThisYear and baseyear == now.year-dateRange.yearCross(): # don't consider this year when looking for records for last event, because this year is not over continue expectedDayCount = 0 observedDayCount = 0 for day in dayRange: expectedDayCount += 1 if day in data: observedDayCount += 1 else: continue if baseyear in firstByYear: # We already figured out which day was first continue vals = {} for fieldName, fieldCall in fieldDict.items(): try: vals[fieldName] = fieldCall(data[day], day) vals[fieldName+'Flag'] = '"' + fieldCall.getFlag(data[day]) + '"' except KeyError: #print day, 'KeyError' vals[fieldName] = None skip=False usedVals={} for fieldName in fieldDict.keys(): if fieldName in referencedValues: if vals[fieldName] is None: #print 'Skipping {} because {} is None'.format(day, fieldName) skip=True break usedVals[fieldName] = vals[fieldName] if skip: continue #resolvedExpr = expr.format(**vals) #print(vals) #val = eval(expr) val = eval(expr, vals) #if True: #day == datetime.date(2015,10,17): #print day, resolvedExpr, val if val is True: dayOffset = (day - starttime).days firstByYear[baseyear] = dayOffset break observedPercent = observedDayCount * 100 / expectedDayCount if observedPercent < 85 and baseyear != eThisYear: print('Skipping {baseyear} because it only had {observedPercent:.1f}% of the days'.format(**locals())) if baseyear in firstByYear: firstByYear.pop(baseyear) elif baseyear not in firstByYear and baseyear != eThisYear: print('Event did not happen during {}.'.format(baseyear)) datacache.cacheThis(cityName, baseyear, '{}.{}.{}.{}'.format(name,order,str(dateRange),expr), firstByYear.get(baseyear, None)) yearByFirst = reverseDict(firstByYear) #for offset in sorted(yearByFirst.keys()): # for year in yearByFirst[offset]: # print datetime.date(year, dateRange.startMonth, dateRange.startDay) + datetime.timedelta(offset) #print yearByFirst verbose = False if verboseIfInDateRange == None: verbose = True elif eThisYear in firstByYear: thisYearFirstDayOffset = firstByYear[eThisYear] firstThisYearDate = ( datetime.date(eThisYear, dateRange.startMonth, dateRange.startDay) + datetime.timedelta(thisYearFirstDayOffset) ) if ( firstThisYearDate >= verboseIfInDateRange[0] and firstThisYearDate <= verboseIfInDateRange[1] ): verbose = True (earliest, secondEarliest, *ignore) = sorted(filter(lambda y: y!=now.year-dateRange.yearCross(), yearByFirst.keys()))[:2] + [None,None] (secondLatest, latest, *ignore) = sorted(filter(lambda y: y!=now.year-dateRange.yearCross(), yearByFirst.keys()))[-2:] + [None,None] earliestYears = lookupEmptyListIfNone(yearByFirst, earliest) secondEarliestYears = lookupEmptyListIfNone(yearByFirst, secondEarliest) latestYears = lookupEmptyListIfNone(yearByFirst, latest) secondLatestYears = lookupEmptyListIfNone(yearByFirst, secondLatest) earliestDates=[] latestDates=[] if verbose: earliestDates=showRecords(name, 'earliest', earliestYears, firstByYear, dateRange) showRecords(name, '2nd earliest', secondEarliestYears, firstByYear, dateRange) showRecords(name, '2nd latest', secondLatestYears, firstByYear, dateRange) latestDates=showRecords(name, 'latest', latestYears, firstByYear, dateRange) avgKeys = filter(lambda t: t in firstByYear, range(eThisYear-30, eThisYear)) offsets = sorted([firstByYear[a] for a in avgKeys]) avg = calcAvg(offsets) median = calcMedian(offsets) avgFirst = calcDate(eThisYear, dateRange.startMonth, dateRange.startDay, avg) medianFirst = calcDate(eThisYear, dateRange.startMonth, dateRange.startDay, median) if verbose: print("average {name} is {avgFirst}".format(**locals())) print("median {name} is {medianFirst}".format(**locals())) ret = None if eThisYear in firstByYear: thisYearFirst = firstByYear[eThisYear] countEarlier = 0 countEqual = 0 countLater = 0 recentCountEarlier = [] recentCountEqual = [] recentCountLater = [] for first in sorted(yearByFirst.keys()): if first < thisYearFirst: countEarlier += len(yearByFirst[first]) recentCountEarlier += filter(lambda y: y<eThisYear and y > eThisYear-31, yearByFirst[first]) elif first == thisYearFirst: countEqual += len(yearByFirst[first]) - 1 #Subtract the current year recentCountEqual += filter(lambda y: y<eThisYear and y > eThisYear-31, yearByFirst[first]) else: countLater += len(yearByFirst[first]) recentCountLater += filter(lambda y: y<eThisYear and y > eThisYear-31, yearByFirst[first]) totalCount = countEarlier + countEqual + countLater totalRecentCount = len(recentCountEarlier) + len(recentCountEqual) + len(recentCountLater) firstThisYear = ( datetime.date(eThisYear, dateRange.startMonth, dateRange.startDay) + datetime.timedelta(thisYearFirst) ) if verbose: print('%s %s was %s' % (now.year, name, firstThisYear )) print('%d%% of last %d years were earlier.' % (round(countEarlier * 100.0 / totalCount), totalCount)) print('%d%% of last %d years were the same.' % (round(countEqual * 100.0 / totalCount), totalCount)) print('%d%% of last %d years were later.' % (round(countLater * 100.0 / totalCount), totalCount)) print('%d of last %d years were earlier.' % (len(recentCountEarlier), totalRecentCount), sorted(recentCountEarlier)) print('%d of last %d years were the same.' % (len(recentCountEqual), totalRecentCount), sorted(recentCountEqual)) print('%d of last %d years were later.' % (len(recentCountLater), totalRecentCount), sorted(recentCountLater)) ret = firstThisYear, medianFirst, earliestDates, latestDates else: print('Not showing this year because eThisYear="{}" and firstByYear="{}"'.format(eThisYear, firstByYear)) if len(yearByFirst) == 0: # There's nothing to plot, so don't even bother trying return ret plotDataOtherYears = [] plotDataThisYear = [] dateLabels = [] for key in sorted(yearByFirst.keys()): if eThisYear in yearByFirst[key]: plotDataThisYear.append((key, len(yearByFirst[key]), '#%s' % ','.join(map(str, yearByFirst[key])) )) else: plotDataOtherYears.append((key, len(yearByFirst[key]), '#%s' % ','.join(map(str, yearByFirst[key])))) histogramFname = '%s/svg/%s' % (cityName, name.replace(' ', '_')) if ret != None: ret = ret + (histogramFname,) plot = gnuplot.Plot(histogramFname) dateMin = min(yearByFirst.keys()) dateMax = max(yearByFirst.keys()) plotDateMin = dateMin-1 #int(dateMin - (dateMax - dateMin)*.25) plotDateMax = dateMax+1 #int(dateMax + (dateMax - dateMin)*.25) for dayOffset in range(plotDateMin, plotDateMax+1, 1): date = datetime.date(now.year, dateRange.startMonth, dateRange.startDay) + datetime.timedelta(dayOffset) if date.day % 5 == 1 and date.day != 31: dateLabels.append('"%s" %d' % (date.strftime('%b/%d'), dayOffset)) ylabel = 'Number of years when %s happened on this day' % name if yaxisLabel != None: ylabel = yaxisLabel plot.open(title='%s in %s' % (name, cityName.capitalize()), ylabel=ylabel, xmin=plotDateMin, xmax=plotDateMax, ymin=0, ymax=max(len(a) for a in yearByFirst.values())+1, xtics=dateLabels, xticsRotate=45, margins=[7,8,2,5]) plot.addLine(gnuplot.Line('Other years', plotDataOtherYears, plot='boxes')) plot.addLine(gnuplot.Line('This year', plotDataThisYear, plot='boxes')) plot.plot() plot.close() print('') plotdata = [] for (year, plotCount) in firstByYear.items(): if plotCount != None: plotdata.append( (year, plotCount, ( '#' +str(datetime.date(year, dateRange.startMonth, dateRange.startDay) + datetime.timedelta(plotCount))))) ytics = [] for m in range(1,13): for d in (1,10,20): thisDate = datetime.date(2015,m,d) dayOffset = (thisDate - datetime.date(2015, dateRange.startMonth, dateRange.startDay)).days ytics.append('"%s" %d' % (thisDate.strftime('%b %d'), dayOffset)) #print(tuple(t[:2] for t in plotdata)) lineFit = linear.linearTrend(tuple(t[:2] for t in plotdata)) plot = gnuplot.Plot("%s/svg/%s_%s-%u_%s.line" % (cityName, name.replace(' ', '_'), monthName(dateRange.startMonth), dateRange.startDay, dateRange.lastDay().replace(' ', '-') ), yaxis=2) plot.open(title='%s %s between %s %u and %s' % (cityName.capitalize(), name, monthName(dateRange.startMonth), dateRange.startDay, dateRange.lastDay() ), ytics=ytics) plot.addLine(gnuplot.Line("Linear", lineFit, lineColour='green', plot='lines')) plot.addLine(gnuplot.Line("Date", plotdata, lineColour='purple')) plot.plot() plot.close() return ret