def thisMonthOverTime(month):
for fname in ['MEAN_TEMP']:
findex = daily.fields._fields.index(fname)
data = []
for year in range(min(cityData.keys()),2010):
avg = filterAverage(YearMonthFilter(year, month), findex)
if avg != None:
data.append((year,avg.average))
lineFit = linear.linearTrend(data)
plot = gnuplot.Plot("ThisMonthOverTime_%s_%s" % (fname, monthName(month)), yaxis=2)
plot.open()
plot.addLine(gnuplot.Line(fname, data, lineColour='blue'))
plot.addLine(gnuplot.Line('linear', lineFit, lineColour='green'))
plot.plot()
plot.close()
for fname in ['TOTAL_SNOW_CM', 'TOTAL_PRECIP_MM']:
findex = daily.fields._fields.index(fname)
data = []
for year in range(min(cityData.keys()),2010):
data.append((year, filterSum(YearMonthFilter(year, month), findex).total))
lineFit = linear.linearTrend(data)
plot = gnuplot.Plot("ThisMonthOverTime_%s_%s" % (fname, monthName(month)), yaxis=2)
plot.open()
plot.addLine(gnuplot.Line(fname, data, lineColour='blue'))
plot.addLine(gnuplot.Line('linear', lineFit, lineColour='green'))
plot.plot()
plot.close()
def plotline(valueByYear, filename, chartTitle, yaxisLabel):
data = sorted(tuple(valueByYear.items()))
if len(data) > 1:
lineFit = linear.linearTrend(tuple((a[0], float(a[1])) for a in data))
plot = gnuplot.Plot(filename, yaxis=2)
plot.open(title=chartTitle, ylabel=yaxisLabel)
if len(data) > 1:
plot.addLine(
gnuplot.Line("Linear", lineFit, lineColour='green', plot='lines'))
plot.addLine(gnuplot.Line("Temp", data, lineColour='purple'))
plot.plot()
plot.close()
return plot.fname
print "Sun: %4d %2dC %3d" % (year, t, tempSunHours)
if tempSunHours > recordVal:
sunRecord[t] = (year, tempSunHours)
#
thisyearsnowy = totalHours(weather.hourly.WEATHER, Snowy(year))
snowData.append((year, thisyearsnowy))
print "Snow: %d\t%d" % (year, thisyearsnowy)
print sunRecord
sunLineFit = {10: [], 15: [], 20: [], 25: [], 30: []}
for t in sunData:
sunLineFit[t] = linear.linearTrend(sunData[t])
snowlineFit = linear.linearTrend(snowData)
plot = gnuplot.Plot("ottawa/SunnyHours", yaxis=2)
plot.open(ymin=0)
for t in sunData:
plot.addLine(gnuplot.Line("Trend %d" % t, sunLineFit[t], plot='lines'))
plot.addLine(gnuplot.Line("Sun %d" % t, sunData[t]))
plot.plot()
plot.close()
plot = gnuplot.Plot("ottawa/SnowHours", yaxis=2)
plot.open(ymin=0)
plot.addLine(
gnuplot.Line("Trend", snowlineFit, lineColour='green', plot='lines'))
def count(cityName,
data,
expr,
name,
verbose=True,
skipIncomplete=False,
lineChart=False):
countByYear = {}
thisYearCount = 0
for baseyear in range(data.minYear, data.maxYear - YEAR_CROSS() + 1):
countByYear[baseyear] = None
countThisYear = 0
starttime = datetime.datetime(baseyear, START_MONTH, START_DAY)
endtime = datetime.datetime(baseyear + YEAR_CROSS(), END_MONTH,
END_DAY)
hourdiff = (endtime - starttime).days * 24
hoursWithObservations = 0
thisRunLen = 0
#print baseyear
for day in hourly.hourRange(starttime, endtime):
dayValues = data.get(day, None)
val = None
flag = ''
if dayValues != None:
#print day, dayValues
val, flag = expr(dayValues)
if val != None:
hoursWithObservations += 1
if val and not (skipIncomplete and 'I' in flag):
countThisYear += 1
if baseyear + YEAR_CROSS() == now.year:
if verbose:
print "Count: %s" % day, float(val), flag
if (day.date() == min(
endtime.date() - datetime.timedelta(1),
datetime.date.today())):
# We only care about this year's count if the final day of
# this year actually met the criteria
thisYearCount = countThisYear
else:
#if baseyear == 2014:
# print "Skip: %s" % day
thisRunLen = 0
if (hoursWithObservations >= hourdiff * 85 /
100): # or baseyear + YEAR_CROSS() == now.year:
countByYear[baseyear] = countThisYear
if verbose:
print baseyear, countThisYear, hoursWithObservations
else:
# skip years with fewer than 90% of days recorded
if verbose:
print("Skipping %d because it has only %u/%u records" %
(baseyear, hoursWithObservations, hourdiff))
#del countByYear[now.year]
(fewest, most) = keyOfMinMaxValue(countByYear)
if verbose:
print "fewest %s was %s (%d)" % (name, fewest, countByYear[fewest[0]])
print "most %s was %s (%d)" % (name, most, countByYear[most[0]])
eThisYear = now.year
if YEAR_CROSS():
eThisYear -= 1
avgKeys = filter(lambda t: t in countByYear,
range(eThisYear - 30, eThisYear))
counts = [countByYear[a] for a in avgKeys]
counts = filter(lambda t: t != None, counts)
avg = float(sum(counts)) / len(counts)
if verbose:
print "average %s is %f" % (name, avg)
#print thisYearCount
mostSince = None
prevRecordYears = []
prevRecord = 0
if eThisYear in countByYear:
#thisYearCount = countByYear[eThisYear]
if thisYearCount > 0:
for year in reversed(sorted(countByYear.keys())):
if mostSince == None and year != eThisYear and countByYear[
year] >= thisYearCount:
mostSince = year
if year != eThisYear:
if countByYear[year] > prevRecord:
prevRecord = countByYear[year]
prevRecordYears = [year]
elif countByYear[year] == prevRecord:
prevRecordYears.append(year)
if verbose:
print now.year, name, "was", countByYear[eThisYear]
if mostSince != None:
print "Most since %d." % (mostSince)
print(
"%d of past %d years had more %s" %
(len(filter(lambda t: countByYear[t] > thisYearCount,
avgKeys)), len(avgKeys), name),
filter(lambda t: countByYear[t] > thisYearCount, avgKeys))
print(
"%d of past %d years had equal %s" %
(len(filter(lambda t: countByYear[t] == thisYearCount,
avgKeys)), len(avgKeys), name),
filter(lambda t: countByYear[t] == thisYearCount, avgKeys))
print(
"%d of past %d years had fewer %s" %
(len(filter(lambda t: countByYear[t] < thisYearCount,
avgKeys)), len(avgKeys), name),
filter(lambda t: countByYear[t] < thisYearCount, avgKeys))
print
if lineChart:
plotdata = []
for (year, plotCount) in countByYear.iteritems():
if plotCount != None:
plotdata.append((year, plotCount))
lineFit = linear.linearTrend(plotdata)
plot = gnuplot.Plot(
"%s/svg/%s_count_%s-%u_before_%s" %
(cityName, name.replace(' ', '_'), monthName(START_MONTH),
START_DAY, lastDay().replace(' ', '-')),
yaxis=2)
plot.open(title='%s number of %s between %s %u and %s' %
(cityName.capitalize(), name, monthName(START_MONTH),
START_DAY, lastDay()))
plot.addLine(
gnuplot.Line("Linear", lineFit, lineColour='green', plot='lines'))
plot.addLine(gnuplot.Line("Count", plotdata, lineColour='purple'))
plot.plot()
plot.close()
else:
barPlotData = makeFit(countByYear, 75)
filename = ("%s/svg/%s_count_%s-%u_to_%s_bar" %
(cityName, name.replace(' ', '_'), monthName(START_MONTH),
START_DAY, lastDay().replace(' ', '-')))
plotdict(barPlotData,
filename=filename,
chartTitle=('%s number of %s between %s %u and %s' %
(cityName.capitalize(), name,
monthName(START_MONTH), START_DAY, lastDay())),
yaxisLabel='',
thisYear=eThisYear,
ymin=0)
return (filename, thisYearCount, mostSince, prevRecord, prevRecordYears)
city = sys.argv[1]
cityData = daily.load(city)
for field in [
FractionVal(daily.MAX_TEMP, "high temperature"),
FractionVal(daily.MIN_TEMP, 'low temperature'),
FractionVal(daily.AVG_WIND, 'wind'),
Avg(daily.MIN_TEMP, daily.MAX_TEMP, 'average temperature')
]:
fname = field.name
print(fname)
data = getAnnualValues(cityData, field, False)
try:
lineFit = linear.linearTrend(data)
except ZeroDivisionError:
print(data)
raise
for i in range(len(data)):
year, val = data[i]
lfitv = lineFit[i][1]
print(year, float(val), lfitv)
plot = gnuplot.Plot("%s/svg/Annual_%s" % (city, fname), yaxis=2)
plot.open(title='%s %s per year' % (city.capitalize(), field.title),
ylabel='%s in %s' % (field.title, field.units))
plot.addLine(
gnuplot.Line("Linear", lineFit, lineColour='green', plot='lines'))
plot.addLine(gnuplot.Line("Temp", data, lineColour='purple'))
def first(cityName,
expr,
name,
dateRange,
excludeThisYear=False,
order="first",
yaxisLabel=None,
run=1,
limitToMostRecentYears=None,
verboseIfInDateRange=None,
):
data = daily.load(cityName)
if limitToMostRecentYears != None:
ndata = daily.Data()
yearsAgo = datetime.date(now.year-limitToMostRecentYears, now.month, now.day)
for key in data:
if key >= yearsAgo:
ndata[key] = data[key]
data = ndata
fieldDict = {
'min': Value(daily.MIN_TEMP),
'max': Value(daily.MAX_TEMP),
'tempSpan': ValueDiff(daily.MAX_TEMP, daily.MIN_TEMP),
'rain': Value(daily.TOTAL_RAIN_MM),
'humidex': Value(daily.MAX_HUMIDEX),
'snow': Value(daily.TOTAL_SNOW_CM),
'snowpack': ValueEmptyZero(daily.SNOW_ON_GRND_CM),
'windgust': Value(daily.SPD_OF_MAX_GUST_KPH),
'wind': Value(daily.AVG_WIND),
'windchill': Value(daily.MIN_WINDCHILL),
'avgWindchill': Value(daily.MIN_WINDCHILL),
}
fieldValues = fieldDict.values()
referencedValues=[]
for fieldName in fieldDict.keys():
if '{'+fieldName+'}' in expr:
referencedValues.append(fieldName)
firstByYear = {}
eThisYear = now.year
endTimeThisYear = datetime.date(now.year, dateRange.endMonth, dateRange.endDay)
if dateRange.yearCross() and now < endTimeThisYear:
eThisYear -= 1
for baseyear in range(data.minYear, data.maxYear-dateRange.todayIsYearCross()+1):
try:
dayOffset = datacache.readCache(cityName, baseyear,
'{}.{}.{}.{}'.format(name,order,str(dateRange),expr))
if dayOffset != None:
firstByYear[baseyear] = dayOffset
continue
except datacache.NotInCache:
pass
starttime = datetime.date(baseyear, dateRange.startMonth, dateRange.startDay)
endtime = datetime.date(baseyear+dateRange.yearCross(), dateRange.endMonth, dateRange.endDay)
#print baseyear, starttime, endtime
dayRange = daily.dayRange(starttime,endtime)
if order=="last":
dayRange = daily.dayRange(endtime-datetime.timedelta(1), starttime-datetime.timedelta(1), -1)
if excludeThisYear and baseyear == now.year-dateRange.yearCross():
# don't consider this year when looking for records for last event, because this year is not over
continue
expectedDayCount = 0
observedDayCount = 0
for day in dayRange:
expectedDayCount += 1
if day in data:
observedDayCount += 1
else:
continue
if baseyear in firstByYear:
# We already figured out which day was first
continue
vals = {}
for fieldName, fieldCall in fieldDict.items():
try:
vals[fieldName] = fieldCall(data[day], day)
vals[fieldName+'Flag'] = '"' + fieldCall.getFlag(data[day]) + '"'
except KeyError:
#print day, 'KeyError'
vals[fieldName] = None
skip=False
usedVals={}
for fieldName in fieldDict.keys():
if fieldName in referencedValues:
if vals[fieldName] is None:
#print 'Skipping {} because {} is None'.format(day, fieldName)
skip=True
break
usedVals[fieldName] = vals[fieldName]
if skip:
continue
#resolvedExpr = expr.format(**vals)
#print(vals)
#val = eval(expr)
val = eval(expr, vals)
#if True: #day == datetime.date(2015,10,17):
#print day, resolvedExpr, val
if val is True:
dayOffset = (day - starttime).days
firstByYear[baseyear] = dayOffset
break
observedPercent = observedDayCount * 100 / expectedDayCount
if observedPercent < 85 and baseyear != eThisYear:
print('Skipping {baseyear} because it only had {observedPercent:.1f}% of the days'.format(**locals()))
if baseyear in firstByYear:
firstByYear.pop(baseyear)
elif baseyear not in firstByYear and baseyear != eThisYear:
print('Event did not happen during {}.'.format(baseyear))
datacache.cacheThis(cityName, baseyear,
'{}.{}.{}.{}'.format(name,order,str(dateRange),expr),
firstByYear.get(baseyear, None))
yearByFirst = reverseDict(firstByYear)
#for offset in sorted(yearByFirst.keys()):
# for year in yearByFirst[offset]:
# print datetime.date(year, dateRange.startMonth, dateRange.startDay) + datetime.timedelta(offset)
#print yearByFirst
verbose = False
if verboseIfInDateRange == None:
verbose = True
elif eThisYear in firstByYear:
thisYearFirstDayOffset = firstByYear[eThisYear]
firstThisYearDate = (
datetime.date(eThisYear, dateRange.startMonth, dateRange.startDay)
+ datetime.timedelta(thisYearFirstDayOffset) )
if ( firstThisYearDate >= verboseIfInDateRange[0]
and firstThisYearDate <= verboseIfInDateRange[1] ):
verbose = True
(earliest, secondEarliest, *ignore) = sorted(filter(lambda y: y!=now.year-dateRange.yearCross(), yearByFirst.keys()))[:2] + [None,None]
(secondLatest, latest, *ignore) = sorted(filter(lambda y: y!=now.year-dateRange.yearCross(), yearByFirst.keys()))[-2:] + [None,None]
earliestYears = lookupEmptyListIfNone(yearByFirst, earliest)
secondEarliestYears = lookupEmptyListIfNone(yearByFirst, secondEarliest)
latestYears = lookupEmptyListIfNone(yearByFirst, latest)
secondLatestYears = lookupEmptyListIfNone(yearByFirst, secondLatest)
earliestDates=[]
latestDates=[]
if verbose:
earliestDates=showRecords(name, 'earliest', earliestYears, firstByYear, dateRange)
showRecords(name, '2nd earliest', secondEarliestYears, firstByYear, dateRange)
showRecords(name, '2nd latest', secondLatestYears, firstByYear, dateRange)
latestDates=showRecords(name, 'latest', latestYears, firstByYear, dateRange)
avgKeys = filter(lambda t: t in firstByYear, range(eThisYear-30, eThisYear))
offsets = sorted([firstByYear[a] for a in avgKeys])
avg = calcAvg(offsets)
median = calcMedian(offsets)
avgFirst = calcDate(eThisYear, dateRange.startMonth, dateRange.startDay, avg)
medianFirst = calcDate(eThisYear, dateRange.startMonth, dateRange.startDay, median)
if verbose:
print("average {name} is {avgFirst}".format(**locals()))
print("median {name} is {medianFirst}".format(**locals()))
ret = None
if eThisYear in firstByYear:
thisYearFirst = firstByYear[eThisYear]
countEarlier = 0
countEqual = 0
countLater = 0
recentCountEarlier = []
recentCountEqual = []
recentCountLater = []
for first in sorted(yearByFirst.keys()):
if first < thisYearFirst:
countEarlier += len(yearByFirst[first])
recentCountEarlier += filter(lambda y: y<eThisYear and y > eThisYear-31, yearByFirst[first])
elif first == thisYearFirst:
countEqual += len(yearByFirst[first]) - 1 #Subtract the current year
recentCountEqual += filter(lambda y: y<eThisYear and y > eThisYear-31, yearByFirst[first])
else:
countLater += len(yearByFirst[first])
recentCountLater += filter(lambda y: y<eThisYear and y > eThisYear-31, yearByFirst[first])
totalCount = countEarlier + countEqual + countLater
totalRecentCount = len(recentCountEarlier) + len(recentCountEqual) + len(recentCountLater)
firstThisYear = ( datetime.date(eThisYear, dateRange.startMonth, dateRange.startDay)
+ datetime.timedelta(thisYearFirst) )
if verbose:
print('%s %s was %s' % (now.year, name, firstThisYear ))
print('%d%% of last %d years were earlier.'
% (round(countEarlier * 100.0 / totalCount), totalCount))
print('%d%% of last %d years were the same.'
% (round(countEqual * 100.0 / totalCount), totalCount))
print('%d%% of last %d years were later.'
% (round(countLater * 100.0 / totalCount), totalCount))
print('%d of last %d years were earlier.'
% (len(recentCountEarlier), totalRecentCount), sorted(recentCountEarlier))
print('%d of last %d years were the same.'
% (len(recentCountEqual), totalRecentCount), sorted(recentCountEqual))
print('%d of last %d years were later.'
% (len(recentCountLater), totalRecentCount), sorted(recentCountLater))
ret = firstThisYear, medianFirst, earliestDates, latestDates
else:
print('Not showing this year because eThisYear="{}" and firstByYear="{}"'.format(eThisYear, firstByYear))
if len(yearByFirst) == 0:
# There's nothing to plot, so don't even bother trying
return ret
plotDataOtherYears = []
plotDataThisYear = []
dateLabels = []
for key in sorted(yearByFirst.keys()):
if eThisYear in yearByFirst[key]:
plotDataThisYear.append((key, len(yearByFirst[key]), '#%s' % ','.join(map(str, yearByFirst[key])) ))
else:
plotDataOtherYears.append((key, len(yearByFirst[key]), '#%s' % ','.join(map(str, yearByFirst[key]))))
histogramFname = '%s/svg/%s' % (cityName, name.replace(' ', '_'))
if ret != None:
ret = ret + (histogramFname,)
plot = gnuplot.Plot(histogramFname)
dateMin = min(yearByFirst.keys())
dateMax = max(yearByFirst.keys())
plotDateMin = dateMin-1 #int(dateMin - (dateMax - dateMin)*.25)
plotDateMax = dateMax+1 #int(dateMax + (dateMax - dateMin)*.25)
for dayOffset in range(plotDateMin, plotDateMax+1, 1):
date = datetime.date(now.year, dateRange.startMonth, dateRange.startDay) + datetime.timedelta(dayOffset)
if date.day % 5 == 1 and date.day != 31:
dateLabels.append('"%s" %d' % (date.strftime('%b/%d'), dayOffset))
ylabel = 'Number of years when %s happened on this day' % name
if yaxisLabel != None:
ylabel = yaxisLabel
plot.open(title='%s in %s' % (name, cityName.capitalize()),
ylabel=ylabel,
xmin=plotDateMin,
xmax=plotDateMax,
ymin=0,
ymax=max(len(a) for a in yearByFirst.values())+1,
xtics=dateLabels, xticsRotate=45,
margins=[7,8,2,5])
plot.addLine(gnuplot.Line('Other years', plotDataOtherYears, plot='boxes'))
plot.addLine(gnuplot.Line('This year', plotDataThisYear, plot='boxes'))
plot.plot()
plot.close()
print('')
plotdata = []
for (year, plotCount) in firstByYear.items():
if plotCount != None:
plotdata.append(
(year, plotCount,
( '#'
+str(datetime.date(year, dateRange.startMonth, dateRange.startDay)
+ datetime.timedelta(plotCount)))))
ytics = []
for m in range(1,13):
for d in (1,10,20):
thisDate = datetime.date(2015,m,d)
dayOffset = (thisDate - datetime.date(2015, dateRange.startMonth, dateRange.startDay)).days
ytics.append('"%s" %d' % (thisDate.strftime('%b %d'), dayOffset))
#print(tuple(t[:2] for t in plotdata))
lineFit = linear.linearTrend(tuple(t[:2] for t in plotdata))
plot = gnuplot.Plot("%s/svg/%s_%s-%u_%s.line"
% (cityName, name.replace(' ', '_'),
monthName(dateRange.startMonth), dateRange.startDay,
dateRange.lastDay().replace(' ', '-') ),
yaxis=2)
plot.open(title='%s %s between %s %u and %s'
% (cityName.capitalize(), name,
monthName(dateRange.startMonth), dateRange.startDay,
dateRange.lastDay() ),
ytics=ytics)
plot.addLine(gnuplot.Line("Linear", lineFit, lineColour='green', plot='lines'))
plot.addLine(gnuplot.Line("Date", plotdata, lineColour='purple'))
plot.plot()
plot.close()
return ret