def thisMonthOverTime(month):
for fname in ['MEAN_TEMP']:
findex = daily.fields._fields.index(fname)
data = []
for year in range(min(cityData.keys()),2010):
avg = filterAverage(YearMonthFilter(year, month), findex)
if avg != None:
data.append((year,avg.average))
lineFit = linear.linearTrend(data)
plot = gnuplot.Plot("ThisMonthOverTime_%s_%s" % (fname, monthName(month)), yaxis=2)
plot.open()
plot.addLine(gnuplot.Line(fname, data, lineColour='blue'))
plot.addLine(gnuplot.Line('linear', lineFit, lineColour='green'))
plot.plot()
plot.close()
for fname in ['TOTAL_SNOW_CM', 'TOTAL_PRECIP_MM']:
findex = daily.fields._fields.index(fname)
data = []
for year in range(min(cityData.keys()),2010):
data.append((year, filterSum(YearMonthFilter(year, month), findex).total))
lineFit = linear.linearTrend(data)
plot = gnuplot.Plot("ThisMonthOverTime_%s_%s" % (fname, monthName(month)), yaxis=2)
plot.open()
plot.addLine(gnuplot.Line(fname, data, lineColour='blue'))
plot.addLine(gnuplot.Line('linear', lineFit, lineColour='green'))
plot.plot()
plot.close()
def annualOrderedByMonth(monthFilter, field, cumulative):
valByYears = yearDatas(monthFilter, field, now.year, cumulative)
plotDataOtherYears = []
plotDataThisYear = []
plotTics = []
plotIndex = 0
divider = 1
ylabel = field.units
if max(valByYears.values()) > 10000:
divider = 1000.0
ylabel = '1000s of %s' % field.units
valByYears = makeFit(valByYears, 75)
plotIndex = 0
last30Years = []
yearsByVal = reverseDictionary(valByYears)
for val in sorted(yearsByVal.keys()):
yearsWithThisVal = yearsByVal[val]
for plotYear in yearsWithThisVal:
if now.year == plotYear:
plotDataThisYear.append((plotIndex, val/divider))
else:
plotDataOtherYears.append((plotIndex, val/divider))
plotTics.append('"%s" %d' % (plotYear, plotIndex))
plotIndex += 1
if (plotYear > now.year - 31) and (plotYear < now.year):
last30Years.append(val/divider)
avg = sum(last30Years) / len(last30Years)
std = numpy.std([float(a) for a in last30Years])
#print
if len(yearsByVal.keys()) > 0:
plot = gnuplot.Plot('%s/svg/yearOrdering.%s.%s' % (cityName,
monthFilter.filenamePart,
field.title),
yaxis=2)
legend='on left'
if max(yearsByVal.keys()) < 0:
legend='bottom right'
ymin = None
if cumulative:
ymin = 0
plot.open(xtics=plotTics, xticsRotate=90, xticsFont='Arial,10', legend=legend,
ymin=ymin,
title='%s %s for %s' % (cityName.capitalize(),
field.title.lower(),
monthFilter.chartTitle),
margins=[6,8,2,3],
ylabel=ylabel
)
plot.addLine(gnuplot.Line('30-year std-dev', ((-1, avg-std), (plotIndex, avg-std), (plotIndex, avg+std), (-1, avg+std)), lineColour='#e3e3e3', plot='filledcurves'))
plot.addLine(gnuplot.Line('Other years', plotDataOtherYears, plot='boxes'))
plot.addLine(gnuplot.Line('This year', plotDataThisYear, plot='boxes'))
plot.addLine(gnuplot.Line('30-Year Average', ((0, avg), (plotIndex-1, avg))))
plot.plot()
plot.close()
def plotline(valueByYear, filename, chartTitle, yaxisLabel):
data = sorted(tuple(valueByYear.items()))
#lineFit = linear.linearTrend(data)
plot = gnuplot.Plot(filename, yaxis=2)
plot.open(title=chartTitle, ylabel=yaxisLabel)
#plot.addLine(gnuplot.Line("Linear", lineFit, lineColour='green', plot='lines'))
plot.addLine(gnuplot.Line("Temp", data, lineColour='purple'))
plot.plot()
plot.close()
return plot.fname
def plotMonthly(name, filterFunc):
sunnyHoursByMonth = totalHoursByMonth(filterFunc)
#
monthVals = {}
#
currentYearVals = []
lastYearVals = []
#
for year in sorted(sunnyHoursByMonth.keys()):
for month in sorted(sunnyHoursByMonth[year].keys()):
if month not in monthVals:
monthVals[month] = []
sunnySum = sunnyHoursByMonth[year][month]
average = sunnySum.average()
if year != now.year:
monthVals[month].append(average)
if year == now.year:
print "%d/%02d: %.2f" % (year, month, average)
currentYearVals.append((month, average))
elif year == now.year - 1:
lastYearVals.append((month, average))
#
plotAvg = []
plotMin = []
plotMax = []
#
for month in sorted(monthVals.keys()):
Avg = sum(monthVals[month]) / len(monthVals[month])
Min = min(monthVals[month])
Max = max(monthVals[month])
print "%d: avg=%.2f, min=%.2f, max=%.2f" % (month, Avg, Min, Max)
plotAvg.append((month, Avg))
plotMin.append((month, Min))
plotMax.append((month, Max))
#
#
plot = gnuplot.Plot('ottawa/%s.%s' % (now.year, name), yaxis=2)
plot.open(xtics=[
'"Jan" 1', '"Feb" 2', '"Mar" 3', '"Apr" 4', '"May" 5', '"Jun" 6',
'"Jul" 7', '"Aug" 8', '"Sep" 9', '"Oct" 10', '"Nov" 11', '"Dec" 12'
])
plot.addLine(
gnuplot.Line('%d' % now.year, currentYearVals, lineColour='purple'))
plot.addLine(
gnuplot.Line('%d' % (now.year - 1), lastYearVals, lineColour='orange'))
plot.addLine(gnuplot.Line('Average', plotAvg, lineColour='green'))
plot.addLine(gnuplot.Line('Min', plotMin, lineColour='blue'))
plot.addLine(gnuplot.Line('Max', plotMax, lineColour='red'))
plot.plot()
plot.close()
def plotDifference(city1Name, city2Name, field, cumulative):
city1ByYear = getValuesByYear(city1Name, field, cumulative)
city2ByYear = getValuesByYear(city2Name, field, cumulative)
#
bothYears = sorted(
list(sets.Set(city1ByYear.keys()) & sets.Set(city2ByYear.keys())))
#
plotData = []
for year in bothYears:
diff = city2ByYear[year] - city1ByYear[year]
print "%d: %.1f" % (year, diff)
plotData.append((year, diff))
#
plot = gnuplot.Plot("%s_minus_%s_Annual_%s" %
('farm', 'ottawa', daily.fields[field].name),
yaxis=2)
plot.open()
plot.addLine(
gnuplot.Line("Temp", plotData, lineColour='purple', plot='boxes'))
plot.plot()
plot.close()
#
thisyearsnowy = totalHours(weather.hourly.WEATHER, Snowy(year))
snowData.append((year, thisyearsnowy))
print "Snow: %d\t%d" % (year, thisyearsnowy)
print sunRecord
sunLineFit = {10: [], 15: [], 20: [], 25: [], 30: []}
for t in sunData:
sunLineFit[t] = linear.linearTrend(sunData[t])
snowlineFit = linear.linearTrend(snowData)
plot = gnuplot.Plot("ottawa/SunnyHours", yaxis=2)
plot.open(ymin=0)
for t in sunData:
plot.addLine(gnuplot.Line("Trend %d" % t, sunLineFit[t], plot='lines'))
plot.addLine(gnuplot.Line("Sun %d" % t, sunData[t]))
plot.plot()
plot.close()
plot = gnuplot.Plot("ottawa/SnowHours", yaxis=2)
plot.open(ymin=0)
plot.addLine(
gnuplot.Line("Trend", snowlineFit, lineColour='green', plot='lines'))
plot.addLine(gnuplot.Line("Snow", snowData, lineColour='purple'))
plot.plot()
plot.close()
def count(cityName,
data,
expr,
name,
verbose=True,
skipIncomplete=False,
lineChart=False):
countByYear = {}
thisYearCount = 0
for baseyear in range(data.minYear, data.maxYear - YEAR_CROSS() + 1):
countByYear[baseyear] = None
countThisYear = 0
starttime = datetime.datetime(baseyear, START_MONTH, START_DAY)
endtime = datetime.datetime(baseyear + YEAR_CROSS(), END_MONTH,
END_DAY)
hourdiff = (endtime - starttime).days * 24
hoursWithObservations = 0
thisRunLen = 0
#print baseyear
for day in hourly.hourRange(starttime, endtime):
dayValues = data.get(day, None)
val = None
flag = ''
if dayValues != None:
#print day, dayValues
val, flag = expr(dayValues)
if val != None:
hoursWithObservations += 1
if val and not (skipIncomplete and 'I' in flag):
countThisYear += 1
if baseyear + YEAR_CROSS() == now.year:
if verbose:
print "Count: %s" % day, float(val), flag
if (day.date() == min(
endtime.date() - datetime.timedelta(1),
datetime.date.today())):
# We only care about this year's count if the final day of
# this year actually met the criteria
thisYearCount = countThisYear
else:
#if baseyear == 2014:
# print "Skip: %s" % day
thisRunLen = 0
if (hoursWithObservations >= hourdiff * 85 /
100): # or baseyear + YEAR_CROSS() == now.year:
countByYear[baseyear] = countThisYear
if verbose:
print baseyear, countThisYear, hoursWithObservations
else:
# skip years with fewer than 90% of days recorded
if verbose:
print("Skipping %d because it has only %u/%u records" %
(baseyear, hoursWithObservations, hourdiff))
#del countByYear[now.year]
(fewest, most) = keyOfMinMaxValue(countByYear)
if verbose:
print "fewest %s was %s (%d)" % (name, fewest, countByYear[fewest[0]])
print "most %s was %s (%d)" % (name, most, countByYear[most[0]])
eThisYear = now.year
if YEAR_CROSS():
eThisYear -= 1
avgKeys = filter(lambda t: t in countByYear,
range(eThisYear - 30, eThisYear))
counts = [countByYear[a] for a in avgKeys]
counts = filter(lambda t: t != None, counts)
avg = float(sum(counts)) / len(counts)
if verbose:
print "average %s is %f" % (name, avg)
#print thisYearCount
mostSince = None
prevRecordYears = []
prevRecord = 0
if eThisYear in countByYear:
#thisYearCount = countByYear[eThisYear]
if thisYearCount > 0:
for year in reversed(sorted(countByYear.keys())):
if mostSince == None and year != eThisYear and countByYear[
year] >= thisYearCount:
mostSince = year
if year != eThisYear:
if countByYear[year] > prevRecord:
prevRecord = countByYear[year]
prevRecordYears = [year]
elif countByYear[year] == prevRecord:
prevRecordYears.append(year)
if verbose:
print now.year, name, "was", countByYear[eThisYear]
if mostSince != None:
print "Most since %d." % (mostSince)
print(
"%d of past %d years had more %s" %
(len(filter(lambda t: countByYear[t] > thisYearCount,
avgKeys)), len(avgKeys), name),
filter(lambda t: countByYear[t] > thisYearCount, avgKeys))
print(
"%d of past %d years had equal %s" %
(len(filter(lambda t: countByYear[t] == thisYearCount,
avgKeys)), len(avgKeys), name),
filter(lambda t: countByYear[t] == thisYearCount, avgKeys))
print(
"%d of past %d years had fewer %s" %
(len(filter(lambda t: countByYear[t] < thisYearCount,
avgKeys)), len(avgKeys), name),
filter(lambda t: countByYear[t] < thisYearCount, avgKeys))
print
if lineChart:
plotdata = []
for (year, plotCount) in countByYear.iteritems():
if plotCount != None:
plotdata.append((year, plotCount))
lineFit = linear.linearTrend(plotdata)
plot = gnuplot.Plot(
"%s/svg/%s_count_%s-%u_before_%s" %
(cityName, name.replace(' ', '_'), monthName(START_MONTH),
START_DAY, lastDay().replace(' ', '-')),
yaxis=2)
plot.open(title='%s number of %s between %s %u and %s' %
(cityName.capitalize(), name, monthName(START_MONTH),
START_DAY, lastDay()))
plot.addLine(
gnuplot.Line("Linear", lineFit, lineColour='green', plot='lines'))
plot.addLine(gnuplot.Line("Count", plotdata, lineColour='purple'))
plot.plot()
plot.close()
else:
barPlotData = makeFit(countByYear, 75)
filename = ("%s/svg/%s_count_%s-%u_to_%s_bar" %
(cityName, name.replace(' ', '_'), monthName(START_MONTH),
START_DAY, lastDay().replace(' ', '-')))
plotdict(barPlotData,
filename=filename,
chartTitle=('%s number of %s between %s %u and %s' %
(cityName.capitalize(), name,
monthName(START_MONTH), START_DAY, lastDay())),
yaxisLabel='',
thisYear=eThisYear,
ymin=0)
return (filename, thisYearCount, mostSince, prevRecord, prevRecordYears)
def createPlot(plotData, city, name, yaxisLabel, fieldEnglishName, units,
plotDate, plotZeros, verbose):
yearByValue = {}
valueByYear = {}
for year in plotData:
#
value = plotData[year]
#if value == None:
# value = 0
if (value not in yearByValue):
yearByValue[value] = [year]
else:
yearByValue[value].append(year)
valueByYear[year] = value
#print year, value
top10Years = valuesOfHighest10Keys(yearByValue)
bottom10Years = valuesOfLowest10Keys(yearByValue)
allfname = plotdict(
valueByYear,
filename='{city}/svg/{name}.all'.format(**locals()),
chartTitle='%s daily %s for %s' %
(city.title(), fieldEnglishName, plotDate.strftime('%b %d')),
yaxisLabel=yaxisLabel,
thisYear=plotDate.year,
plotZeros=True,
showAverage=False,
)
lineFname = plotline(
valueByYear,
filename='{city}/svg/{name}.line'.format(**locals()),
chartTitle='%s daily %s for %s' %
(city.title(), fieldEnglishName, plotDate.strftime('%b %d')),
yaxisLabel=yaxisLabel,
)
top10Fname = plotdict(
filterDict(valueByYear, top10Years),
filename='%s/svg/%s.top10' % (city, name),
chartTitle='Top 10 %s daily %ss for %s' %
(city.title(), fieldEnglishName, plotDate.strftime('%b %d')),
yaxisLabel=yaxisLabel,
thisYear=plotDate.year,
plotZeros=True,
showAverage=False,
)
print(filterDict(valueByYear, bottom10Years))
bottom10Fname = plotdict(
filterDict(valueByYear, bottom10Years),
filename='%s/svg/%s.bottom10' % (city, name),
chartTitle='Bottom 10 %s daily %ss for %s' %
(city.title(), fieldEnglishName, plotDate.strftime('%b %d')),
yaxisLabel=yaxisLabel,
thisYear=plotDate.year,
plotZeros=True,
showAverage=False,
)
values = sorted(yearByValue.keys())
#
chartTicks = []
chartDataOtherYears = []
chartDataThisYear = []
chartIndex = 0
count = 0
recentValues = []
maxValue = max(values)
for value in values:
# number of years with this value
thiscount = len(yearByValue[value])
#
recentYearsWithThisValue = tuple(
filter(lambda y: y < plotDate.year and y > plotDate.year - 31,
yearByValue[value]))
if verbose:
lrecent = len(recentValues)
lrecentVal = len(recentYearsWithThisValue)
isMedian = (lrecent <= 15) and (lrecent + lrecentVal > 15)
#print lrecent, lrecentVal, map(float, recentValues)
print("{}-{}) {:.1f}{}, {}{}".format(count + 1,
len(valueByYear) - count,
value, units,
yearByValue[value],
["", "*** median"][isMedian]))
recentValues += [value] * len(recentYearsWithThisValue)
count += thiscount
#
if plotZeros == False and value == 0:
# We've been told to skip zeros, so we don't plot them
continue
for year in yearByValue[value]:
if year == plotDate.year:
chartDataThisYear.append((chartIndex, value, '# %d' % year))
else:
chartDataOtherYears.append((chartIndex, value, '# %d' % year))
chartTicks.append('"%d" %d' % (year, chartIndex))
chartIndex += 1
legend = 'on left'
if maxValue < 0:
legend = 'on right bottom'
bmargin = 5
plot = gnuplot.Plot('%s/svg/%s.yearOrdering' % (city, name),
yaxis=2,
output='svg')
#
plot.open(
title='%s daily %s for %s' %
(city.title(), fieldEnglishName, plotDate.strftime('%b %d')),
xtics=chartTicks,
xticsRotate=90,
xticsFont='Arial,10',
legend=legend,
margins=[6, 8, 2, bmargin],
ylabel=yaxisLabel, # ymin=0,
xmin=-1,
xmax=chartIndex)
plot.addLine(
gnuplot.Line('Other years',
chartDataOtherYears,
plot='boxes',
lineColour='0x6495ED'))
plot.addLine(
gnuplot.Line('This year',
chartDataThisYear,
plot='boxes',
lineColour='0x556B2F'))
plot.plot()
plot.close()
return lineFname, bottom10Fname, top10Fname
def annualPlots(city, year):
for field in [
Avg(daily.MAX_TEMP, daily.MIN_TEMP, "Temperature"),
FractionVal(daily.MAX_TEMP, "Max temp"),
FractionVal(daily.MIN_TEMP, "Min temp"),
FractionVal(daily.MEAN_TEMP, "Mean temp"),
FractionVal(daily.MEAN_HUMIDITY, "Mean humidity"),
FractionVal(daily.SNOW_ON_GRND_CM, "Snow-on-the-ground"),
FractionVal(daily.AVG_WIND, "Wind"),
FractionVal(daily.AVG_DEWPOINT, "Mean dewpoint"),
]:
print(field.name)
current = []
average = []
lowest = []
highest = []
stdLow = []
stdHigh = []
for month in range(1, 13):
thisyear = filterAverage(YearMonthFilter(year, month), field)
normal = normalMonthlyAverage(
year, YearMonthFilter(year=None, month=month), field)
average.append("%d %.1f" % (month, normal.average))
stdLow.append("%d %.1f" % (month, normal.average - normal.std))
stdHigh.append("%d %.1f" % (month, normal.average + normal.std))
lowest.append("%d %.1f" % (month, normal.minimum))
highest.append("%d %.1f" % (month, normal.maximum))
if thisyear != None and thisyear.count >= 15:
current.append("%d %.1f" % (month, thisyear.average))
units = field.units
print(
"%10s-%10s: %+6.2f%s (%d: %+6.2f%s, normal: %+6.2f%s, extremeMin: %+6.2f%s %s, extremeMax: %+6.2f%s %s)"
% (monthName(month), field.name,
(thisyear.average - normal.average), units, year,
thisyear.average, units, normal.average, units,
normal.minimum, units, normal.minYear, normal.maximum,
units, normal.maxYear))
stdData = stdLow + list(reversed(stdHigh))
plot = gnuplot.Plot(
"%s/svg/%d_%s" %
(city, year, field.name.replace(' ', '_').replace('℃', 'C')),
yaxis=2)
plot.open(title="%s %s" % (city, field.title),
xtics=[
'"Jan" 1', '"Feb" 2', '"Mar" 3', '"Apr" 4', '"May" 5',
'"Jun" 6', '"Jul" 7', '"Aug" 8', '"Sep" 9', '"Oct" 10',
'"Nov" 11', '"Dec" 12'
])
plot.addLine(
gnuplot.Line('30-year std-dev',
stdData,
lineColour='#e3e3e3',
plot='filledcurves'))
plot.addLine(
gnuplot.Line("30-year normal", average, lineColour='#007700'))
plot.addLine(gnuplot.Line("Record min", lowest, lineColour='blue'))
plot.addLine(gnuplot.Line("Record max", highest, lineColour='red'))
plot.addLine(gnuplot.Line("%d" % year, current, lineColour='purple'))
plot.plot()
plot.close()
for field in (FractionVal(daily.TOTAL_SNOW_CM, "Snowfall"),
FractionVal(daily.TOTAL_RAIN_MM, "Rainfall"),
FractionVal(daily.TOTAL_PRECIP_MM, "Precip")):
print(field.name)
print("%s\t%s\t%s\t%s\t%s" %
("Month", "%d" % year, "Average", "Record max", "Record min"))
current = []
average = []
lowest = []
highest = []
stdLow = []
stdHigh = []
for month in range(1, 13):
thisyear = filterSum(YearMonthFilter(year, month), field)
normal = normalMonthlySum(year, YearMonthFilter(year, month),
field)
if normal != None:
average.append((month, normal.average))
stdLow.append("%d %.1f" % (month, normal.average - normal.std))
stdHigh.append("%d %.1f" %
(month, normal.average + normal.std))
lowest.append((month, normal.minimum))
highest.append((month, normal.maximum))
if thisyear.count > 13:
current.append((month, thisyear.total))
units = field.units
print(
"%10s-%10s: %+6.1f%s (%d: %+6.1f%s, normal: %+6.1f%s, extremeMin: %+6.1f%s %s, extremeMax: %+6.1f%s %s)"
% (monthName(month), field.name,
(thisyear.total - normal.average), units, year,
thisyear.total, units, normal.average, units,
normal.minimum, units, normal.minYear,
normal.maximum, units, normal.maxYear))
if (len(current) > 0 and len(average) > 0 and len(lowest) > 0
and len(highest) > 0):
stdData = stdLow + list(reversed(stdHigh))
plot = gnuplot.Plot(
"%s/svg/%d_%s" %
(city, year, field.name.replace(' ', '_').replace('℃', 'C')),
yaxis=2)
plot.open(title="%s %s" % (city, field.title),
xtics=[
'"Jan" 1', '"Feb" 2', '"Mar" 3', '"Apr" 4',
'"May" 5', '"Jun" 6', '"Jul" 7', '"Aug" 8',
'"Sep" 9', '"Oct" 10', '"Nov" 11', '"Dec" 12'
])
plot.addLine(
gnuplot.Line('30-year std-dev',
stdData,
lineColour='#e3e3e3',
plot='filledcurves'))
plot.addLine(
gnuplot.Line("30-year normal", average, lineColour='#007700'))
plot.addLine(gnuplot.Line("Record min", lowest, lineColour='blue'))
plot.addLine(gnuplot.Line("Record max", highest, lineColour='red'))
plot.addLine(
gnuplot.Line("%d" % year, current, lineColour='purple'))
plot.plot()
plot.close()
fname = field.name
print(fname)
data = getAnnualValues(cityData, field, False)
try:
lineFit = linear.linearTrend(data)
except ZeroDivisionError:
print(data)
raise
for i in range(len(data)):
year, val = data[i]
lfitv = lineFit[i][1]
print(year, float(val), lfitv)
plot = gnuplot.Plot("%s/svg/Annual_%s" % (city, fname), yaxis=2)
plot.open(title='%s %s per year' % (city.capitalize(), field.title),
ylabel='%s in %s' % (field.title, field.units))
plot.addLine(
gnuplot.Line("Linear", lineFit, lineColour='green', plot='lines'))
plot.addLine(gnuplot.Line("Temp", data, lineColour='purple'))
plot.plot()
plot.close()
for field in [
FractionVal(daily.TOTAL_SNOW_CM, 'snow'),
FractionVal(daily.SNOW_ON_GRND_CM, 'snowpack'),
FractionVal(daily.TOTAL_RAIN_MM, 'rain'),
FractionVal(daily.TOTAL_PRECIP_MM, 'precipitation')
]:
fname = field.name
co2Fit.append((year, co2fv))
if i > 0:
progressivePrediction = linear.linearExtra(data[:(i + 1)])
progressiveLineFit.append((year + 1, progressivePrediction))
progressiveError.append(abs(progressivePrediction - val))
lineFitError.append(abs(lfitv - val))
sameError.append(abs(float(data[i - 1][1] - val)))
co2FitError.append(abs(co2fv - val))
print year, float(val), lfitv
co2ErrorBarSize = sorted(co2FitError)[len(co2FitError) * 95 / 100]
co2ErrorBarPlot = []
for year, val in co2Fit:
co2ErrorBarPlot.append((year, val, co2ErrorBarSize))
plot = gnuplot.Plot("%s/Annual_Prediction_%s" % (city, fname), yaxis=2)
plot.open(title='%s %s per year' % (city.capitalize(), field.title),
ylabel='%s in %s' % (field.title, field.units))
plot.addLine(
gnuplot.Line("Linear", lineFit, lineColour='green', plot='lines'))
plot.addLine(
gnuplot.Line("Extrapolation",
progressiveLineFit,
lineColour='red',
plot='lines'))
plot.addLine(gnuplot.Line("Co2", co2Fit, lineColour='blue'))
plot.addLine(
gnuplot.Line("Co2 Error",
co2ErrorBarPlot,
lineColour='blue',
plot='errorbars'))
def plotdict(valueByYear,
filename,
chartTitle,
yaxisLabel,
thisYear,
plotZeros=True,
output='svg',
ymin=None,
showAverage=True):
yearByValue = reverseDict(valueByYear)
values = sorted(yearByValue.keys())
#
chartTicks = []
chartDataOtherYears = []
chartDataThisYear = []
chartIndex = 0
maxValue = max(values)
if showAverage:
recentYears = tuple(
filter(lambda t: t in valueByYear, range(thisYear - 30, thisYear)))
recentValues = [valueByYear[t] for t in recentYears]
recentAvg = sum(recentValues) / len(recentValues)
recentStd = numpy.std(tuple(map(float, recentValues)))
longestXTick = 0
for value in values:
# number of years with this value
#print 'val="%s"' % value
thiscount = len(yearByValue[value])
#
if plotZeros == False and value == 0:
# We've been told to skip zeros, so we don't plot them
continue
for year in yearByValue[value]:
if year == thisYear:
chartDataThisYear.append((chartIndex, value, '# ' + str(year)))
else:
chartDataOtherYears.append(
(chartIndex, value, '# ' + str(year)))
xtick = str(year)
longestXTick = max(longestXTick, len(xtick))
chartTicks.append('"{date}" {index}'.format(date=xtick,
index=chartIndex))
chartIndex += 1
legend = 'on left'
if maxValue < 0:
legend = 'on right bottom'
bmargin = 2 + longestXTick // 2
plot = gnuplot.Plot(filename, yaxis=2, output=output)
#
#, xticsFont='Arial,10'
plot.open(title=chartTitle,
xtics=chartTicks,
xticsRotate=90,
xticsFont='Arial,20',
legend=legend,
margins=[6, 8, 2, bmargin],
ylabel=yaxisLabel,
ymin=ymin,
xmin=-1,
xmax=chartIndex)
if showAverage:
plot.addLine(
gnuplot.Line('30-year 2*std-dev',
((-1, recentAvg - recentStd * 2),
(chartIndex, recentAvg - recentStd * 2),
(chartIndex, recentAvg + recentStd * 2),
(-1, recentAvg + recentStd * 2)),
lineColour='#f1f1f1',
plot='filledcurves'))
plot.addLine(
gnuplot.Line('30-year std-dev',
((-1, recentAvg - recentStd),
(chartIndex, recentAvg - recentStd),
(chartIndex, recentAvg + recentStd),
(-1, recentAvg + recentStd)),
lineColour='#e3e3e3',
plot='filledcurves'))
plot.addLine(
gnuplot.Line('30-year average',
((-1, recentAvg), (chartIndex, recentAvg)),
lineColour='0x000000'))
plot.addLine(
gnuplot.Line('Other years',
chartDataOtherYears,
plot='boxes',
lineColour='0x6495ED'))
plot.addLine(
gnuplot.Line('This year',
chartDataThisYear,
plot='boxes',
lineColour='0x556B2F'))
plot.plot()
plot.close()
return plot.fname
for field in [
#FractionVal(daily.MIN_WINDCHILL, "windchill"),
Avg(daily.MAX_TEMP, daily.MIN_TEMP, 'average temperature'),
FractionVal(daily.AVG_WIND, 'wind'),
#FractionVal(daily.SNOW_ON_GRND_CM, 'average snow depth'),
]:
fname = field.name
print(fname)
data = getAnnualValues(cityData, field, False)
assert len(data)
for year, val in data:
print('%u: %.1f' % (year, val))
lineFit = linear.linearTrend(data)
plot = gnuplot.Plot("%s/svg/Winter_%s" % (city, fname), yaxis=2)
plot.open(title='%s %s per winter (%s %u to %s %u)'
% (city.capitalize(), field.title,
monthName(START_MONTH), START_DAY,
monthName(END_MONTH), END_DAY) )
plot.addLine(gnuplot.Line("Linear", lineFit, lineColour='green', plot='lines'))
plot.addLine(gnuplot.Line("Temp", data, lineColour='purple'))
plot.plot()
plot.close()
print('30-year average: %.1f' % normalYearlyAverage(cityData, 2014, field))
for field in [FractionVal(daily.TOTAL_SNOW_CM, 'snow')]:
fname = field.name
print(fname)
data = getAnnualValues(cityData, field, True)
assert len(data)
def first(cityName,
expr,
name,
dateRange,
excludeThisYear=False,
order="first",
yaxisLabel=None,
run=1,
limitToMostRecentYears=None,
verboseIfInDateRange=None,
):
data = daily.load(cityName)
if limitToMostRecentYears != None:
ndata = daily.Data()
yearsAgo = datetime.date(now.year-limitToMostRecentYears, now.month, now.day)
for key in data:
if key >= yearsAgo:
ndata[key] = data[key]
data = ndata
fieldDict = {
'min': Value(daily.MIN_TEMP),
'max': Value(daily.MAX_TEMP),
'tempSpan': ValueDiff(daily.MAX_TEMP, daily.MIN_TEMP),
'rain': Value(daily.TOTAL_RAIN_MM),
'humidex': Value(daily.MAX_HUMIDEX),
'snow': Value(daily.TOTAL_SNOW_CM),
'snowpack': ValueEmptyZero(daily.SNOW_ON_GRND_CM),
'windgust': Value(daily.SPD_OF_MAX_GUST_KPH),
'wind': Value(daily.AVG_WIND),
'windchill': Value(daily.MIN_WINDCHILL),
'avgWindchill': Value(daily.MIN_WINDCHILL),
}
fieldValues = fieldDict.values()
referencedValues=[]
for fieldName in fieldDict.keys():
if '{'+fieldName+'}' in expr:
referencedValues.append(fieldName)
firstByYear = {}
eThisYear = now.year
endTimeThisYear = datetime.date(now.year, dateRange.endMonth, dateRange.endDay)
if dateRange.yearCross() and now < endTimeThisYear:
eThisYear -= 1
for baseyear in range(data.minYear, data.maxYear-dateRange.todayIsYearCross()+1):
try:
dayOffset = datacache.readCache(cityName, baseyear,
'{}.{}.{}.{}'.format(name,order,str(dateRange),expr))
if dayOffset != None:
firstByYear[baseyear] = dayOffset
continue
except datacache.NotInCache:
pass
starttime = datetime.date(baseyear, dateRange.startMonth, dateRange.startDay)
endtime = datetime.date(baseyear+dateRange.yearCross(), dateRange.endMonth, dateRange.endDay)
#print baseyear, starttime, endtime
dayRange = daily.dayRange(starttime,endtime)
if order=="last":
dayRange = daily.dayRange(endtime-datetime.timedelta(1), starttime-datetime.timedelta(1), -1)
if excludeThisYear and baseyear == now.year-dateRange.yearCross():
# don't consider this year when looking for records for last event, because this year is not over
continue
expectedDayCount = 0
observedDayCount = 0
for day in dayRange:
expectedDayCount += 1
if day in data:
observedDayCount += 1
else:
continue
if baseyear in firstByYear:
# We already figured out which day was first
continue
vals = {}
for fieldName, fieldCall in fieldDict.items():
try:
vals[fieldName] = fieldCall(data[day], day)
vals[fieldName+'Flag'] = '"' + fieldCall.getFlag(data[day]) + '"'
except KeyError:
#print day, 'KeyError'
vals[fieldName] = None
skip=False
usedVals={}
for fieldName in fieldDict.keys():
if fieldName in referencedValues:
if vals[fieldName] is None:
#print 'Skipping {} because {} is None'.format(day, fieldName)
skip=True
break
usedVals[fieldName] = vals[fieldName]
if skip:
continue
#resolvedExpr = expr.format(**vals)
#print(vals)
#val = eval(expr)
val = eval(expr, vals)
#if True: #day == datetime.date(2015,10,17):
#print day, resolvedExpr, val
if val is True:
dayOffset = (day - starttime).days
firstByYear[baseyear] = dayOffset
break
observedPercent = observedDayCount * 100 / expectedDayCount
if observedPercent < 85 and baseyear != eThisYear:
print('Skipping {baseyear} because it only had {observedPercent:.1f}% of the days'.format(**locals()))
if baseyear in firstByYear:
firstByYear.pop(baseyear)
elif baseyear not in firstByYear and baseyear != eThisYear:
print('Event did not happen during {}.'.format(baseyear))
datacache.cacheThis(cityName, baseyear,
'{}.{}.{}.{}'.format(name,order,str(dateRange),expr),
firstByYear.get(baseyear, None))
yearByFirst = reverseDict(firstByYear)
#for offset in sorted(yearByFirst.keys()):
# for year in yearByFirst[offset]:
# print datetime.date(year, dateRange.startMonth, dateRange.startDay) + datetime.timedelta(offset)
#print yearByFirst
verbose = False
if verboseIfInDateRange == None:
verbose = True
elif eThisYear in firstByYear:
thisYearFirstDayOffset = firstByYear[eThisYear]
firstThisYearDate = (
datetime.date(eThisYear, dateRange.startMonth, dateRange.startDay)
+ datetime.timedelta(thisYearFirstDayOffset) )
if ( firstThisYearDate >= verboseIfInDateRange[0]
and firstThisYearDate <= verboseIfInDateRange[1] ):
verbose = True
(earliest, secondEarliest, *ignore) = sorted(filter(lambda y: y!=now.year-dateRange.yearCross(), yearByFirst.keys()))[:2] + [None,None]
(secondLatest, latest, *ignore) = sorted(filter(lambda y: y!=now.year-dateRange.yearCross(), yearByFirst.keys()))[-2:] + [None,None]
earliestYears = lookupEmptyListIfNone(yearByFirst, earliest)
secondEarliestYears = lookupEmptyListIfNone(yearByFirst, secondEarliest)
latestYears = lookupEmptyListIfNone(yearByFirst, latest)
secondLatestYears = lookupEmptyListIfNone(yearByFirst, secondLatest)
earliestDates=[]
latestDates=[]
if verbose:
earliestDates=showRecords(name, 'earliest', earliestYears, firstByYear, dateRange)
showRecords(name, '2nd earliest', secondEarliestYears, firstByYear, dateRange)
showRecords(name, '2nd latest', secondLatestYears, firstByYear, dateRange)
latestDates=showRecords(name, 'latest', latestYears, firstByYear, dateRange)
avgKeys = filter(lambda t: t in firstByYear, range(eThisYear-30, eThisYear))
offsets = sorted([firstByYear[a] for a in avgKeys])
avg = calcAvg(offsets)
median = calcMedian(offsets)
avgFirst = calcDate(eThisYear, dateRange.startMonth, dateRange.startDay, avg)
medianFirst = calcDate(eThisYear, dateRange.startMonth, dateRange.startDay, median)
if verbose:
print("average {name} is {avgFirst}".format(**locals()))
print("median {name} is {medianFirst}".format(**locals()))
ret = None
if eThisYear in firstByYear:
thisYearFirst = firstByYear[eThisYear]
countEarlier = 0
countEqual = 0
countLater = 0
recentCountEarlier = []
recentCountEqual = []
recentCountLater = []
for first in sorted(yearByFirst.keys()):
if first < thisYearFirst:
countEarlier += len(yearByFirst[first])
recentCountEarlier += filter(lambda y: y<eThisYear and y > eThisYear-31, yearByFirst[first])
elif first == thisYearFirst:
countEqual += len(yearByFirst[first]) - 1 #Subtract the current year
recentCountEqual += filter(lambda y: y<eThisYear and y > eThisYear-31, yearByFirst[first])
else:
countLater += len(yearByFirst[first])
recentCountLater += filter(lambda y: y<eThisYear and y > eThisYear-31, yearByFirst[first])
totalCount = countEarlier + countEqual + countLater
totalRecentCount = len(recentCountEarlier) + len(recentCountEqual) + len(recentCountLater)
firstThisYear = ( datetime.date(eThisYear, dateRange.startMonth, dateRange.startDay)
+ datetime.timedelta(thisYearFirst) )
if verbose:
print('%s %s was %s' % (now.year, name, firstThisYear ))
print('%d%% of last %d years were earlier.'
% (round(countEarlier * 100.0 / totalCount), totalCount))
print('%d%% of last %d years were the same.'
% (round(countEqual * 100.0 / totalCount), totalCount))
print('%d%% of last %d years were later.'
% (round(countLater * 100.0 / totalCount), totalCount))
print('%d of last %d years were earlier.'
% (len(recentCountEarlier), totalRecentCount), sorted(recentCountEarlier))
print('%d of last %d years were the same.'
% (len(recentCountEqual), totalRecentCount), sorted(recentCountEqual))
print('%d of last %d years were later.'
% (len(recentCountLater), totalRecentCount), sorted(recentCountLater))
ret = firstThisYear, medianFirst, earliestDates, latestDates
else:
print('Not showing this year because eThisYear="{}" and firstByYear="{}"'.format(eThisYear, firstByYear))
if len(yearByFirst) == 0:
# There's nothing to plot, so don't even bother trying
return ret
plotDataOtherYears = []
plotDataThisYear = []
dateLabels = []
for key in sorted(yearByFirst.keys()):
if eThisYear in yearByFirst[key]:
plotDataThisYear.append((key, len(yearByFirst[key]), '#%s' % ','.join(map(str, yearByFirst[key])) ))
else:
plotDataOtherYears.append((key, len(yearByFirst[key]), '#%s' % ','.join(map(str, yearByFirst[key]))))
histogramFname = '%s/svg/%s' % (cityName, name.replace(' ', '_'))
if ret != None:
ret = ret + (histogramFname,)
plot = gnuplot.Plot(histogramFname)
dateMin = min(yearByFirst.keys())
dateMax = max(yearByFirst.keys())
plotDateMin = dateMin-1 #int(dateMin - (dateMax - dateMin)*.25)
plotDateMax = dateMax+1 #int(dateMax + (dateMax - dateMin)*.25)
for dayOffset in range(plotDateMin, plotDateMax+1, 1):
date = datetime.date(now.year, dateRange.startMonth, dateRange.startDay) + datetime.timedelta(dayOffset)
if date.day % 5 == 1 and date.day != 31:
dateLabels.append('"%s" %d' % (date.strftime('%b/%d'), dayOffset))
ylabel = 'Number of years when %s happened on this day' % name
if yaxisLabel != None:
ylabel = yaxisLabel
plot.open(title='%s in %s' % (name, cityName.capitalize()),
ylabel=ylabel,
xmin=plotDateMin,
xmax=plotDateMax,
ymin=0,
ymax=max(len(a) for a in yearByFirst.values())+1,
xtics=dateLabels, xticsRotate=45,
margins=[7,8,2,5])
plot.addLine(gnuplot.Line('Other years', plotDataOtherYears, plot='boxes'))
plot.addLine(gnuplot.Line('This year', plotDataThisYear, plot='boxes'))
plot.plot()
plot.close()
print('')
plotdata = []
for (year, plotCount) in firstByYear.items():
if plotCount != None:
plotdata.append(
(year, plotCount,
( '#'
+str(datetime.date(year, dateRange.startMonth, dateRange.startDay)
+ datetime.timedelta(plotCount)))))
ytics = []
for m in range(1,13):
for d in (1,10,20):
thisDate = datetime.date(2015,m,d)
dayOffset = (thisDate - datetime.date(2015, dateRange.startMonth, dateRange.startDay)).days
ytics.append('"%s" %d' % (thisDate.strftime('%b %d'), dayOffset))
#print(tuple(t[:2] for t in plotdata))
lineFit = linear.linearTrend(tuple(t[:2] for t in plotdata))
plot = gnuplot.Plot("%s/svg/%s_%s-%u_%s.line"
% (cityName, name.replace(' ', '_'),
monthName(dateRange.startMonth), dateRange.startDay,
dateRange.lastDay().replace(' ', '-') ),
yaxis=2)
plot.open(title='%s %s between %s %u and %s'
% (cityName.capitalize(), name,
monthName(dateRange.startMonth), dateRange.startDay,
dateRange.lastDay() ),
ytics=ytics)
plot.addLine(gnuplot.Line("Linear", lineFit, lineColour='green', plot='lines'))
plot.addLine(gnuplot.Line("Date", plotdata, lineColour='purple'))
plot.plot()
plot.close()
return ret