def __call__(self, year):
if type(year) != tuple:
year = (year, )
for y in year:
if self.month == 1 and self.day == 1:
yield from daily.dayRange(datetime.date(y, 1, 1),
datetime.date(y + 1, 1, 1))
else:
yield from daily.dayRange(
datetime.date(y, 1, 1),
datetime.date(y, self.month, self.day))
def rawcount(year, expr, dayFilter = None):
ret = SumCount(sum=0, count=0)
#
startDay = datetime.date(year, 1, 1) #January 1st
endDay = datetime.date(year+1, 1, 1) #January 1st
observedDays = 0
totalDays = 0
#
for day in daily.dayRange(startDay, endDay):
if dayFilter != None and not dayFilter(day):
continue
#
try:
dayValues = weatherData[day]
except KeyError:
#print("Skipping day", day)
continue
#
totalDays += 1
result = expr(dayValues)
if result != None:
observedDays += 1
if result is not False and result is not None:
ret.count += 1
ret.sum += result
#print("Day {} does meet criteria".format(day, result))
else:
#print("Day {} does not meet criteria".format(day), dayValues)
pass
#
if year != now.year and observedDays * 5 < totalDays * 4:
return None
return ret
def get14Days(startDate):
ret = {}
for date in daily.dayRange(startDate + datetime.timedelta(1),
startDate + datetime.timedelta(15)):
ret[date.strftime('<td class="date">%A<br />%b. ') +
'%d</td>' % date.day] = date
return ret
def call(self, city, sampleDay):
yesterday=sampleDay-datetime.timedelta(days=1)
datemarker = dateMarkerFromDate(sampleDay)
alreadyReportedRecord = self.datemarker.get(datemarker,None)
v = [None]*self.dayCount
for i in range(self.dayCount):
v[i] = self.calcForDays(city, sampleDay, days=i+1, history=False)
if ( i > 1
and v[i] is not None
and self.calcForDays(city, yesterday, days=i, history=False) >= v[i]
):
print("Skipping {sampleDay} because the previous day's record was worse."
.format(**locals()))
v[i] = None
if v.count(None) == len(v):
# We have no observations to go by
return
maxV = [None]*self.dayCount
minV = None
for recordDay in daily.dayRange(yesterday,
min(daily.dataByCity[city].keys())-datetime.timedelta(days=1),
-1):
for i in range(self.dayCount):
if maxV[i] is None and v[i] is not None:
val = self.calcForDays(city, recordDay, days=i+1, history=True)
if val is None:
continue
if val >= v[i]:
recordValue = val
maxV[i] = (recordValue, recordDay)
break
if maxV.count(None) == 0:
break
oldestDate = None
for i in range(len(maxV)):
if v[i] is None:
continue
if maxV[i] is None:
oldestIndex = i
dayCount = i + 1
break
if oldestDate is None or maxV[i][1] < oldestDate:
oldestDate = maxV[i][1]
oldestIndex = i
dayCount = i + 1
if dayCount == 1:
return #There are other tests to catch 1-day records
if alreadyReportedRecord is not None:
alreadyReportedRecord = PreviousReport(value=alreadyReportedRecord[0],
tweet=alreadyReportedRecord[1])
if str(round(float(v[oldestIndex]),1)) == str(float(alreadyReportedRecord.value)):
return
self.maybeTweetRecordSince(city, v[oldestIndex], maxV[oldestIndex],
alreadyReportedRecord,
dayCount, sampleDay, datemarker)
def call(self, city, sampleDay):
datemarker = dateMarkerFromDate(sampleDay)
alreadyReportedRecord = self.datemarker.get(datemarker,None)
try:
dayValues = daily.dataByCity[city][sampleDay]
except KeyError:
return
v = dayValues[self.field.index]
if v is None:
return
flag = dayValues[self.field.index+1]
if self.skipIncomplete and 'I' in flag:
return
if self.skipMinEstimated and self.skipMaxEstimated and 'E' in flag:
return
if alreadyReportedRecord != None:
if v == alreadyReportedRecord.value:
return
maxV = None
minV = None
for recordDay in daily.dayRange(sampleDay-datetime.timedelta(days=1),
min(daily.dataByCity[city].keys())-datetime.timedelta(days=1),
-1):
if self.limitToThisMonth is True and recordDay.month != sampleDay.month:
# This is not the month we were instructed to look at, so just skip it
continue
try:
rv = daily.dataByCity[city][recordDay][self.field.index]
except KeyError:
continue
if rv is None:
continue
if maxV == None and rv >= v:
maxV = (rv, recordDay)
if minV is not None:
break
if minV == None and rv <= v:
minV = (rv, recordDay)
if maxV is not None:
break
if not (self.skipMaxRecords
or (self.skipMaxEstimated and 'H' in flag)
or (self.skipMaxIncomplete and 'I' in flag)
):
hour = findHourFlag(flag)
self.maybeTweetRecordSince(city, v, maxV, alreadyReportedRecord,
hour, sampleDay, datemarker, maxRecord=True)
if not (self.skipMinRecords
or (self.skipMinEstimated and 'H' in flag)
or (self.skipMinIncomplete and 'I' in flag)
):
hour = findHourFlag(flag)
self.maybeTweetRecordSince(city, v, minV, alreadyReportedRecord,
hour, sampleDay, datemarker, maxRecord=False)
def main(city, dayToPlot, recently, thisDayInHistory, field):
dailyData = daily.load(city)
dailyDataClean(dailyData)
if dayToPlot is None:
dayToPlot = datetime.date.today()
if recently:
historyLines = daysRecently(dailyData, dayToPlot, field)
else:
historyLines = daysToday(dailyData, dayToPlot, field)
if len(historyLines) == 1: #Most ever
startDate = dailyData.firstDateWithValue(field.index)
else:
startDate = historyLines[1].date
# - datetime.timedelta(days=(historyLines[0].date - historyLines[1].date).days * .1)
endDate = historyLines[0].date + datetime.timedelta(days=1)
plotData = []
for date in daily.dayRange(startDate,
endDate):
values = dailyData.get(date, None)
if values is None:
continue
plotData.append( (date, values[field.index]) )
style=pygal.style.Style(label_font_size=15, major_label_font_size=20)
date_chart = pygal.DateLine(style=style,
print_values=True,
#truncate_label=1000,
x_label_rotation=80)
date_chart.y_title = '{} ({})'.format(field.englishName,
field.units)
if field.minValue == 0:
date_chart.add(None, plotData, show_dots=False, fill=True)
else:
date_chart.add(None, plotData, show_dots=False)
labelByValue = {}
for history in historyLines[0:2]:
labelByValue[float(history.val)] = history.label
date_chart.add(None,
( ( startDate, history.val ),
( historyLines[0].date, history.val ) ),
formatter = lambda t: labelByValue[t[1]] )
historyType = ''
if thisDayInHistory:
historyType = 'thisDayInHistory'
if recently:
historyType = 'recently'
fname = '{city}/{dayToPlot}.recordSince.{historyType}.png'.format(**locals())
date_chart.render_to_png(fname,
width=1024, height=768)
return fname
def yearlyValues(cityData, year, field):
values = []
for date in daily.dayRange(datetime.date(year, 1, 1),
datetime.date(year + 1, 1, 1)):
#for date in daily.dayRange(datetime.date(year,6,1), datetime.date(year,9,1)):
try:
val = field(cityData[date])
if val != None:
values.append(val)
except KeyError:
pass
return values
def dailyDataClean(data):
for year in range(data.minYear, data.maxYear+1):
for day in daily.dayRange( datetime.date(year, 7, 1),
datetime.date(year, 1, 1),
-1):
if day not in data:
continue
vals = data[day]
if vals.SNOW_ON_GRND_CM is not None and vals.SNOW_ON_GRND_CM > 0:
break
vals = vals._replace(SNOW_ON_GRND_CM = 0)
#print('Replacing missing data {day} with zero'.format(**locals()))
data[day] = vals
for day in daily.dayRange( datetime.date(year, 7, 1),
datetime.date(year, 12, 31),
1):
if day not in data:
continue
vals = data[day]
if vals.SNOW_ON_GRND_CM is not None and vals.SNOW_ON_GRND_CM > 0:
break
vals = vals._replace(SNOW_ON_GRND_CM = 0)
#print('Replacing missing data {day} with zero'.format(**locals()))
data[day] = vals
def __call__(self, year):
#import pdb; pdb.set_trace()
if type(year) != tuple:
year = (year, )
for y in year:
baseDate = datetime.date(y, self.month, 1)
endYear = y
endMonth = self.month + 1
if endMonth > 12:
endMonth = 1
endYear += 1
endDate = datetime.date(endYear, endMonth, 1)
yield from daily.dayRange(baseDate, endDate)
def annualData(rawData, field, plotDate):
dataByYear = {}
#
for year in range(rawData.minYear, rawData.maxYear + 1):
#
nextMonthYear = year
nextMonthMonth = plotDate.month
if nextMonthMonth > 11:
nextMonthMonth = 1
nextMonthYear += 1
for day in daily.dayRange(
datetime.date(year, plotDate.month, 1),
datetime.date(nextMonthYear, nextMonthMonth, 1)):
if day in rawData:
if len(rawData[day][field.index]) > 0:
dataByYear[day] = Fraction(rawData[day][field.index])
return dataByYear
def yearlySum(cityData, year, field):
sum = 0
count = 0
fakeCount = 0
for date in daily.dayRange(START_DATE(year), END_DATE(year)):
#print date
try:
val = field(cityData[date])
if val != None:
sum += val
count += 1
elif cityData[date].MAX_TEMP is not None:
fakeCount += 1
except KeyError:
pass
if count > 30:
count += fakeCount
return (sum, count)
def yearlySum(cityData, year, field):
sum = 0
count = 0
fakeCount = 0
for date in daily.dayRange(datetime.date(year, 1, 1),
datetime.date(year + 1, 1, 1)):
try:
val = field(cityData[date])
if val != None:
sum += val
count += 1
elif len(cityData[date].MAX_TEMP) > 0:
fakeCount += 1
except KeyError:
pass
if count > 30:
count += fakeCount
return (sum, count)
def rawdata(year, expr, dayFilter = None):
ret = []
startDay = datetime.date(year, 1, 1) #January 1st
endDay = datetime.date(year+1, 1, 1) #January 1st
for day in daily.dayRange(startDay, endDay):
if dayFilter != None and not dayFilter(day):
continue
try:
dayValues = weatherData[day]
except KeyError:
continue
result = expr(dayValues)
if result != None:
ret.append(result)
return ret
def yearlySum(cityData, year, field):
#print('yearlySum(', year, field.name, ')')
sum = 0
count = 0
fakeCount = 0
for date in daily.dayRange(datetime.date(year, 1, 1),
datetime.date(year + 1, 1, 1)):
#for date in daily.dayRange(datetime.date(year,6,1), datetime.date(year,9,1)):
try:
val = field(cityData[date])
if val is not None:
sum += val
count += 1
elif cityData[date].MAX_TEMP is not None or cityData[
date].MIN_TEMP is not None:
fakeCount += 1
except KeyError:
pass
#if count > 30:
# count += fakeCount
#assert(count>0)
#print('yearlySum(', year, field.name, ')=', (sum, count))
return (sum, count)
def dayRange(self, city):
startDate = today(city) - datetime.timedelta(days=6)
return daily.dayRange(startDate, today(city) + datetime.timedelta(1))
#!/usr/bin/python3
# -*- coding: utf-8 -*-
import daily
import datetime
field = daily.SNOW_ON_GRND_CM
data = daily.load('ottawa')
winterStartByYear = {}
winterEndByYear = {}
for year in range(data.minYear, data.maxYear + 1):
daysCount = 0
missingDays = 0
days = daily.dayRange(datetime.date(year, 7, 1),
datetime.date(year + 1, 7, 1))
for day in days:
if day not in data or data[day].SNOW_ON_GRND_FLAG == 'M':
if missingDays > 0:
# Too many days missing, start from scratch
daysCount = 0
missingDays = 0
else:
missingDays += 1
daysCount += 1
continue
if len(data[day].SNOW_ON_GRND_CM) > 0:
depth = int(data[day].SNOW_ON_GRND_CM)
if depth >= 1:
daysCount += 1
def count(cityName, data, expr, name, endDate,
minRunLen,
showNth,
verbose=True,
skipIncomplete=False,
):
runByStartDate = {}
currentRunStartDate = None
currentRun = []
todaysRun = 0
for sampleDate in daily.dayRange(min(data.keys()), endDate):
dayValues = data.get(sampleDate, None)
val = None
flag = ''
if dayValues != None:
#print day, dayValues
try:
val = dayValues.eval(expr, {'time': sampleDate})
except AttributeError:
print(dayValues)
raise
if val is not None:
if ( val is False
or ( type(val) is tuple and val[0] is False )
or skipIncomplete and 'I' in flag
):
if len(currentRun) >= minRunLen:
#print currentRunStartDate, len(currentRun), ['%.1f' % a for a in currentRun]
runByStartDate[currentRunStartDate] = currentRun
if sampleDate == endDate - dt.timedelta(1):
print('today breaks a run of %d, starting on %s' % (len(currentRun), currentRunStartDate))
currentRun = []
currentRunStartDate = None
else:
if len(currentRun) == 0:
currentRunStartDate = sampleDate
if type(val) is tuple:
currentRun.append(val[1])
else:
currentRun.append(val)
if sampleDate == endDate - dt.timedelta(1):
todaysRun = len(currentRun)
else:
if len(currentRun) >= minRunLen:
runByStartDate[currentRunStartDate] = currentRun
currentRun = []
currentRunStartDate = None
if len(currentRun) >= minRunLen:
runByStartDate[currentRunStartDate] = currentRun
#print runByStartDate
startDateByRunLen = defaultdict(list)
longestRunByWinter = defaultdict(int)
for startDate, run in sorted(runByStartDate.items()):
runlen = len(run)
winter = winterFromDate(startDate)
longestRunByWinter[winter] = max(runlen, longestRunByWinter[winter])
ds = dateDateStr( (startDate, startDate+dt.timedelta(days=runlen-1)) )
print(ds, len(run), run) #, startDateByRunLen
startDateByRunLen[runlen] = list(reversed(sorted(startDateByRunLen[runlen]+[startDate])))
lengths = sorted(startDateByRunLen.keys())
most = lengths[-1]
lastStartDate = max(runByStartDate.keys())
print('lastStartDate', lastStartDate)
if verbose:
print("most %s was %d (%s)" % (name, most, startDateByRunLen[most]))
nth = 1
for l in reversed(lengths):
for d in startDateByRunLen[l]:
if nth <= showNth or d == lastStartDate:
print('%d) %s (%d)' % (nth, dateDateStr( (d, d+dt.timedelta(days=l-1)) ), l), tuple(runByStartDate[d]))
nth += len(startDateByRunLen[l])
print("today's run was %d" % todaysRun, currentRun)
maxLengthPerModernWinter = tuple(
longestRunByWinter[x] for x in filter(
lambda t: t in range(winterFromDate(now)-30,
winterFromDate(now)),
longestRunByWinter.keys()))
print('30-year average: {:.1f}'
.format(sum(maxLengthPerModernWinter)/len(maxLengthPerModernWinter)))
def __call__(self, year):
if type(year) != tuple:
year = (year, )
for y in year:
yield from daily.dayRange(datetime.date(y, self.month1, self.day1),
datetime.date(y, self.month2, self.day2))
for a in x:
std = std + (a - mean)**2
std = sqrt(std / float(n - 1))
return mean, std
weatherData = daily.load('farm')
rawData = {}
for year in range(weatherData.minYear, weatherData.maxYear + 1):
rawData[year] = []
startDay = datetime.date(year, 1, 1)
endDay = datetime.date(year, 2, 1)
for date in daily.dayRange(startDay, endDay):
try:
dayValues = weatherData[date]
except KeyError:
continue
v = dayValues.MAX_TEMP
if v != None and len(v) > 0:
rawData[year].append(float(v))
yearBySwing = {}
for year in sorted(rawData.keys()):
d = rawData[year]
if len(d) > 0:
swings = []
def __call__(self, year):
if type(year) is int:
year = year,
for y in year:
yield from daily.dayRange(dt.date(y, 7, 1), dt.date(y + 1, 7, 1))
def first(cityName,
expr,
name,
dateRange,
excludeThisYear=False,
order="first",
yaxisLabel=None,
run=1,
limitToMostRecentYears=None,
verboseIfInDateRange=None,
):
data = daily.load(cityName)
if limitToMostRecentYears != None:
ndata = daily.Data()
yearsAgo = datetime.date(now.year-limitToMostRecentYears, now.month, now.day)
for key in data:
if key >= yearsAgo:
ndata[key] = data[key]
data = ndata
fieldDict = {
'min': Value(daily.MIN_TEMP),
'max': Value(daily.MAX_TEMP),
'tempSpan': ValueDiff(daily.MAX_TEMP, daily.MIN_TEMP),
'rain': Value(daily.TOTAL_RAIN_MM),
'humidex': Value(daily.MAX_HUMIDEX),
'snow': Value(daily.TOTAL_SNOW_CM),
'snowpack': ValueEmptyZero(daily.SNOW_ON_GRND_CM),
'windgust': Value(daily.SPD_OF_MAX_GUST_KPH),
'wind': Value(daily.AVG_WIND),
'windchill': Value(daily.MIN_WINDCHILL),
'avgWindchill': Value(daily.MIN_WINDCHILL),
}
fieldValues = fieldDict.values()
referencedValues=[]
for fieldName in fieldDict.keys():
if '{'+fieldName+'}' in expr:
referencedValues.append(fieldName)
firstByYear = {}
eThisYear = now.year
endTimeThisYear = datetime.date(now.year, dateRange.endMonth, dateRange.endDay)
if dateRange.yearCross() and now < endTimeThisYear:
eThisYear -= 1
for baseyear in range(data.minYear, data.maxYear-dateRange.todayIsYearCross()+1):
try:
dayOffset = datacache.readCache(cityName, baseyear,
'{}.{}.{}.{}'.format(name,order,str(dateRange),expr))
if dayOffset != None:
firstByYear[baseyear] = dayOffset
continue
except datacache.NotInCache:
pass
starttime = datetime.date(baseyear, dateRange.startMonth, dateRange.startDay)
endtime = datetime.date(baseyear+dateRange.yearCross(), dateRange.endMonth, dateRange.endDay)
#print baseyear, starttime, endtime
dayRange = daily.dayRange(starttime,endtime)
if order=="last":
dayRange = daily.dayRange(endtime-datetime.timedelta(1), starttime-datetime.timedelta(1), -1)
if excludeThisYear and baseyear == now.year-dateRange.yearCross():
# don't consider this year when looking for records for last event, because this year is not over
continue
expectedDayCount = 0
observedDayCount = 0
for day in dayRange:
expectedDayCount += 1
if day in data:
observedDayCount += 1
else:
continue
if baseyear in firstByYear:
# We already figured out which day was first
continue
vals = {}
for fieldName, fieldCall in fieldDict.items():
try:
vals[fieldName] = fieldCall(data[day], day)
vals[fieldName+'Flag'] = '"' + fieldCall.getFlag(data[day]) + '"'
except KeyError:
#print day, 'KeyError'
vals[fieldName] = None
skip=False
usedVals={}
for fieldName in fieldDict.keys():
if fieldName in referencedValues:
if vals[fieldName] is None:
#print 'Skipping {} because {} is None'.format(day, fieldName)
skip=True
break
usedVals[fieldName] = vals[fieldName]
if skip:
continue
#resolvedExpr = expr.format(**vals)
#print(vals)
#val = eval(expr)
val = eval(expr, vals)
#if True: #day == datetime.date(2015,10,17):
#print day, resolvedExpr, val
if val is True:
dayOffset = (day - starttime).days
firstByYear[baseyear] = dayOffset
break
observedPercent = observedDayCount * 100 / expectedDayCount
if observedPercent < 85 and baseyear != eThisYear:
print('Skipping {baseyear} because it only had {observedPercent:.1f}% of the days'.format(**locals()))
if baseyear in firstByYear:
firstByYear.pop(baseyear)
elif baseyear not in firstByYear and baseyear != eThisYear:
print('Event did not happen during {}.'.format(baseyear))
datacache.cacheThis(cityName, baseyear,
'{}.{}.{}.{}'.format(name,order,str(dateRange),expr),
firstByYear.get(baseyear, None))
yearByFirst = reverseDict(firstByYear)
#for offset in sorted(yearByFirst.keys()):
# for year in yearByFirst[offset]:
# print datetime.date(year, dateRange.startMonth, dateRange.startDay) + datetime.timedelta(offset)
#print yearByFirst
verbose = False
if verboseIfInDateRange == None:
verbose = True
elif eThisYear in firstByYear:
thisYearFirstDayOffset = firstByYear[eThisYear]
firstThisYearDate = (
datetime.date(eThisYear, dateRange.startMonth, dateRange.startDay)
+ datetime.timedelta(thisYearFirstDayOffset) )
if ( firstThisYearDate >= verboseIfInDateRange[0]
and firstThisYearDate <= verboseIfInDateRange[1] ):
verbose = True
(earliest, secondEarliest, *ignore) = sorted(filter(lambda y: y!=now.year-dateRange.yearCross(), yearByFirst.keys()))[:2] + [None,None]
(secondLatest, latest, *ignore) = sorted(filter(lambda y: y!=now.year-dateRange.yearCross(), yearByFirst.keys()))[-2:] + [None,None]
earliestYears = lookupEmptyListIfNone(yearByFirst, earliest)
secondEarliestYears = lookupEmptyListIfNone(yearByFirst, secondEarliest)
latestYears = lookupEmptyListIfNone(yearByFirst, latest)
secondLatestYears = lookupEmptyListIfNone(yearByFirst, secondLatest)
earliestDates=[]
latestDates=[]
if verbose:
earliestDates=showRecords(name, 'earliest', earliestYears, firstByYear, dateRange)
showRecords(name, '2nd earliest', secondEarliestYears, firstByYear, dateRange)
showRecords(name, '2nd latest', secondLatestYears, firstByYear, dateRange)
latestDates=showRecords(name, 'latest', latestYears, firstByYear, dateRange)
avgKeys = filter(lambda t: t in firstByYear, range(eThisYear-30, eThisYear))
offsets = sorted([firstByYear[a] for a in avgKeys])
avg = calcAvg(offsets)
median = calcMedian(offsets)
avgFirst = calcDate(eThisYear, dateRange.startMonth, dateRange.startDay, avg)
medianFirst = calcDate(eThisYear, dateRange.startMonth, dateRange.startDay, median)
if verbose:
print("average {name} is {avgFirst}".format(**locals()))
print("median {name} is {medianFirst}".format(**locals()))
ret = None
if eThisYear in firstByYear:
thisYearFirst = firstByYear[eThisYear]
countEarlier = 0
countEqual = 0
countLater = 0
recentCountEarlier = []
recentCountEqual = []
recentCountLater = []
for first in sorted(yearByFirst.keys()):
if first < thisYearFirst:
countEarlier += len(yearByFirst[first])
recentCountEarlier += filter(lambda y: y<eThisYear and y > eThisYear-31, yearByFirst[first])
elif first == thisYearFirst:
countEqual += len(yearByFirst[first]) - 1 #Subtract the current year
recentCountEqual += filter(lambda y: y<eThisYear and y > eThisYear-31, yearByFirst[first])
else:
countLater += len(yearByFirst[first])
recentCountLater += filter(lambda y: y<eThisYear and y > eThisYear-31, yearByFirst[first])
totalCount = countEarlier + countEqual + countLater
totalRecentCount = len(recentCountEarlier) + len(recentCountEqual) + len(recentCountLater)
firstThisYear = ( datetime.date(eThisYear, dateRange.startMonth, dateRange.startDay)
+ datetime.timedelta(thisYearFirst) )
if verbose:
print('%s %s was %s' % (now.year, name, firstThisYear ))
print('%d%% of last %d years were earlier.'
% (round(countEarlier * 100.0 / totalCount), totalCount))
print('%d%% of last %d years were the same.'
% (round(countEqual * 100.0 / totalCount), totalCount))
print('%d%% of last %d years were later.'
% (round(countLater * 100.0 / totalCount), totalCount))
print('%d of last %d years were earlier.'
% (len(recentCountEarlier), totalRecentCount), sorted(recentCountEarlier))
print('%d of last %d years were the same.'
% (len(recentCountEqual), totalRecentCount), sorted(recentCountEqual))
print('%d of last %d years were later.'
% (len(recentCountLater), totalRecentCount), sorted(recentCountLater))
ret = firstThisYear, medianFirst, earliestDates, latestDates
else:
print('Not showing this year because eThisYear="{}" and firstByYear="{}"'.format(eThisYear, firstByYear))
if len(yearByFirst) == 0:
# There's nothing to plot, so don't even bother trying
return ret
plotDataOtherYears = []
plotDataThisYear = []
dateLabels = []
for key in sorted(yearByFirst.keys()):
if eThisYear in yearByFirst[key]:
plotDataThisYear.append((key, len(yearByFirst[key]), '#%s' % ','.join(map(str, yearByFirst[key])) ))
else:
plotDataOtherYears.append((key, len(yearByFirst[key]), '#%s' % ','.join(map(str, yearByFirst[key]))))
histogramFname = '%s/svg/%s' % (cityName, name.replace(' ', '_'))
if ret != None:
ret = ret + (histogramFname,)
plot = gnuplot.Plot(histogramFname)
dateMin = min(yearByFirst.keys())
dateMax = max(yearByFirst.keys())
plotDateMin = dateMin-1 #int(dateMin - (dateMax - dateMin)*.25)
plotDateMax = dateMax+1 #int(dateMax + (dateMax - dateMin)*.25)
for dayOffset in range(plotDateMin, plotDateMax+1, 1):
date = datetime.date(now.year, dateRange.startMonth, dateRange.startDay) + datetime.timedelta(dayOffset)
if date.day % 5 == 1 and date.day != 31:
dateLabels.append('"%s" %d' % (date.strftime('%b/%d'), dayOffset))
ylabel = 'Number of years when %s happened on this day' % name
if yaxisLabel != None:
ylabel = yaxisLabel
plot.open(title='%s in %s' % (name, cityName.capitalize()),
ylabel=ylabel,
xmin=plotDateMin,
xmax=plotDateMax,
ymin=0,
ymax=max(len(a) for a in yearByFirst.values())+1,
xtics=dateLabels, xticsRotate=45,
margins=[7,8,2,5])
plot.addLine(gnuplot.Line('Other years', plotDataOtherYears, plot='boxes'))
plot.addLine(gnuplot.Line('This year', plotDataThisYear, plot='boxes'))
plot.plot()
plot.close()
print('')
plotdata = []
for (year, plotCount) in firstByYear.items():
if plotCount != None:
plotdata.append(
(year, plotCount,
( '#'
+str(datetime.date(year, dateRange.startMonth, dateRange.startDay)
+ datetime.timedelta(plotCount)))))
ytics = []
for m in range(1,13):
for d in (1,10,20):
thisDate = datetime.date(2015,m,d)
dayOffset = (thisDate - datetime.date(2015, dateRange.startMonth, dateRange.startDay)).days
ytics.append('"%s" %d' % (thisDate.strftime('%b %d'), dayOffset))
#print(tuple(t[:2] for t in plotdata))
lineFit = linear.linearTrend(tuple(t[:2] for t in plotdata))
plot = gnuplot.Plot("%s/svg/%s_%s-%u_%s.line"
% (cityName, name.replace(' ', '_'),
monthName(dateRange.startMonth), dateRange.startDay,
dateRange.lastDay().replace(' ', '-') ),
yaxis=2)
plot.open(title='%s %s between %s %u and %s'
% (cityName.capitalize(), name,
monthName(dateRange.startMonth), dateRange.startDay,
dateRange.lastDay() ),
ytics=ytics)
plot.addLine(gnuplot.Line("Linear", lineFit, lineColour='green', plot='lines'))
plot.addLine(gnuplot.Line("Date", plotdata, lineColour='purple'))
plot.plot()
plot.close()
return ret
def __call__(self, city):
if self.field.name in stations.city[city].skipDailyFields:
# This city doesn't keep track of this information in a useful way.
return
ret = []
startDate = today(city) - datetime.timedelta(days=7)
lyear = 0
for sampleDay in daily.dayRange(startDate, today(city)):
if sampleDay.year != lyear:
monthlyRecords = dailyRecords.getMonthRecords(
city, sampleDay.year, self.field)
lyear = sampleDay.year
datemarker = dateMarkerFromDate(sampleDay)
try:
v = citydata[sampleDay][self.field.index]
if v != None and self.datemarker.get(datemarker, None) != v:
maxr = monthlyRecords.max[sampleDay.month - 1]
minr = monthlyRecords.min[sampleDay.month - 1]
msg = ''
if v > maxr.value and self.datemarker.get(
datemarker, None) != v:
print({
'date': sampleDay,
'month': monthName(sampleDay.month, long=True),
'field': self.field.name,
'value': v,
'units': self.field.htmlunits,
'recordValue': maxr.value,
'recordYear': maxr.year,
'recordMonth': maxr.month,
'recordDay': maxr.day
})
tweet = (
'{date}: {month} record high {field} of {value:.1f}{units} breaks previous record of {recordValue:.1f}{units} from {recordYear}/{recordMonth:02d}/{recordDay:02d}\n'
.format(date=sampleDay,
month=monthName(sampleDay.month,
long=True),
field=self.field.name,
value=float(v),
units=self.field.htmlunits,
recordValue=float(maxr.value),
recordYear=maxr.year,
recordMonth=maxr.month,
recordDay=maxr.day))
msg += tweet
self.datemarker[datemarker] = v, tweet
elif v < minr.value and self.skipMinRecords is False:
print({
'date': sampleDay,
'month': monthName(sampleDay.month, long=True),
'field': self.field.name,
'value': v,
'units': self.field.htmlunits,
'recordValue': minr.value,
'recordYear': minr.year,
'recordMonth': minr.month,
'recordDay': minr.day
})
tweet = (
'{date}: {month} record low {field} of {value:.1f}{units} breaks previous record of {recordValue:.1f}{units} from {recordYear}/{recordMonth:02d}/{recordDay:02d}\n'
.format(date=sampleDay,
month=monthName(sampleDay.month,
long=True),
field=self.field.name,
value=float(v),
units=self.field.htmlunits,
recordValue=float(minr.value),
recordYear=minr.year,
recordMonth=minr.month,
recordDay=minr.day))
msg += tweet
self.datemarker[datemarker] = v, tweet
if len(msg) > 0:
ret.append(
Email(field=self.field,
date=sampleDay,
message=msg))
except KeyError:
pass
return ret
def __call__(self, city):
startDate = today(city) - datetime.timedelta(days=7)
for sampleDay in daily.dayRange(startDate,
today(city) + datetime.timedelta(1)):
endDay = sampleDay + datetime.timedelta(1)
startMonth, startDay = self.startMonthDay(sampleDay.month,
sampleDay.day)
(svgFname, result, mostSince, prevRecord,
prevRecordYears) = (first.dispatch(
city,
firstYear=None,
startMonth=startMonth,
startDay=startDay,
endMonth=endDay.month,
endDay=endDay.day,
expression=self.expr,
verbose=False,
skipIncomplete=self.skipIncomplete))
datemarker = dateMarkerFromDate(sampleDay)
alreadyReportedRecord = self.datemarker.get(datemarker, 0)
if result <= alreadyReportedRecord:
try:
os.unlink(svgFname + '.svg')
except OSError as err:
if err.errno != 2:
raise
else:
dayString = 'Today is'
if sampleDay != today(city):
dayString = '%s %d was' % (monthName(
sampleDay.month), sampleDay.day)
context = '??'
if mostSince != None:
hashtag = ''
if (mostSince <= startDate.year - 20
and (today(city) - sampleDay).days < 2):
hashtag = {
'ottawa': ' #OttNews',
'toronto': ' #ToNews'
}[city]
context = 'Most since %d.%s' % (mostSince, hashtag)
elif len(prevRecordYears) > 0:
context = (
'Broke previous record of %d in %s. #%s' %
(prevRecord, '/'.join(map(str, prevRecordYears)), {
'ottawa': 'OttNews',
'toronto': 'ToNews'
}[city]))
tweet = (
'%s #%s\'s %s day with %s during %s. %s' %
(dayString, stations.city[city].name, nth(result),
self.label, self.timeBinDescription(sampleDay), context))
if len(tweet) > 117:
tweet = tweet.replace(' below ', ' < ')
print(tweet)
if (mostSince == None and len(prevRecordYears) > 0
) or mostSince <= startDate.year - 10:
pngname = svgFname.replace('/svg/', '/') + '.png'
command = 'rsvg-convert -o %s --background-color=white %s.svg' % (
pngname, svgFname)
#print command
assert os.system(command) == 0
#print 'api.PostMedia(%s, %s)' % (repr(tweet), repr(pngname))
if auto:
response = 'y'
else:
response = input('"n" to cancel this tweet: ').strip()
if response == 'y':
delayedTweets.addToEndOfListForCity(
city, tweet, pngname)
if response != 'c':
self.datemarker[datemarker] = result
else:
self.datemarker[datemarker] = result
os.unlink(svgFname + '.svg')
if (city == 'ottawa'
and ((mostSince == None and len(prevRecordYears) > 0)
or mostSince <= startDate.year - 20)):
sendMail.sendMailConsole("*****@*****.**",
alert.address,
subject=alert.title,
text=tweet,
files=[pngname],
auto=auto)