def __call__(self, year): if type(year) != tuple: year = (year, ) for y in year: if self.month == 1 and self.day == 1: yield from daily.dayRange(datetime.date(y, 1, 1), datetime.date(y + 1, 1, 1)) else: yield from daily.dayRange( datetime.date(y, 1, 1), datetime.date(y, self.month, self.day))
def rawcount(year, expr, dayFilter = None): ret = SumCount(sum=0, count=0) # startDay = datetime.date(year, 1, 1) #January 1st endDay = datetime.date(year+1, 1, 1) #January 1st observedDays = 0 totalDays = 0 # for day in daily.dayRange(startDay, endDay): if dayFilter != None and not dayFilter(day): continue # try: dayValues = weatherData[day] except KeyError: #print("Skipping day", day) continue # totalDays += 1 result = expr(dayValues) if result != None: observedDays += 1 if result is not False and result is not None: ret.count += 1 ret.sum += result #print("Day {} does meet criteria".format(day, result)) else: #print("Day {} does not meet criteria".format(day), dayValues) pass # if year != now.year and observedDays * 5 < totalDays * 4: return None return ret
def get14Days(startDate): ret = {} for date in daily.dayRange(startDate + datetime.timedelta(1), startDate + datetime.timedelta(15)): ret[date.strftime('<td class="date">%A<br />%b. ') + '%d</td>' % date.day] = date return ret
def call(self, city, sampleDay): yesterday=sampleDay-datetime.timedelta(days=1) datemarker = dateMarkerFromDate(sampleDay) alreadyReportedRecord = self.datemarker.get(datemarker,None) v = [None]*self.dayCount for i in range(self.dayCount): v[i] = self.calcForDays(city, sampleDay, days=i+1, history=False) if ( i > 1 and v[i] is not None and self.calcForDays(city, yesterday, days=i, history=False) >= v[i] ): print("Skipping {sampleDay} because the previous day's record was worse." .format(**locals())) v[i] = None if v.count(None) == len(v): # We have no observations to go by return maxV = [None]*self.dayCount minV = None for recordDay in daily.dayRange(yesterday, min(daily.dataByCity[city].keys())-datetime.timedelta(days=1), -1): for i in range(self.dayCount): if maxV[i] is None and v[i] is not None: val = self.calcForDays(city, recordDay, days=i+1, history=True) if val is None: continue if val >= v[i]: recordValue = val maxV[i] = (recordValue, recordDay) break if maxV.count(None) == 0: break oldestDate = None for i in range(len(maxV)): if v[i] is None: continue if maxV[i] is None: oldestIndex = i dayCount = i + 1 break if oldestDate is None or maxV[i][1] < oldestDate: oldestDate = maxV[i][1] oldestIndex = i dayCount = i + 1 if dayCount == 1: return #There are other tests to catch 1-day records if alreadyReportedRecord is not None: alreadyReportedRecord = PreviousReport(value=alreadyReportedRecord[0], tweet=alreadyReportedRecord[1]) if str(round(float(v[oldestIndex]),1)) == str(float(alreadyReportedRecord.value)): return self.maybeTweetRecordSince(city, v[oldestIndex], maxV[oldestIndex], alreadyReportedRecord, dayCount, sampleDay, datemarker)
def call(self, city, sampleDay): datemarker = dateMarkerFromDate(sampleDay) alreadyReportedRecord = self.datemarker.get(datemarker,None) try: dayValues = daily.dataByCity[city][sampleDay] except KeyError: return v = dayValues[self.field.index] if v is None: return flag = dayValues[self.field.index+1] if self.skipIncomplete and 'I' in flag: return if self.skipMinEstimated and self.skipMaxEstimated and 'E' in flag: return if alreadyReportedRecord != None: if v == alreadyReportedRecord.value: return maxV = None minV = None for recordDay in daily.dayRange(sampleDay-datetime.timedelta(days=1), min(daily.dataByCity[city].keys())-datetime.timedelta(days=1), -1): if self.limitToThisMonth is True and recordDay.month != sampleDay.month: # This is not the month we were instructed to look at, so just skip it continue try: rv = daily.dataByCity[city][recordDay][self.field.index] except KeyError: continue if rv is None: continue if maxV == None and rv >= v: maxV = (rv, recordDay) if minV is not None: break if minV == None and rv <= v: minV = (rv, recordDay) if maxV is not None: break if not (self.skipMaxRecords or (self.skipMaxEstimated and 'H' in flag) or (self.skipMaxIncomplete and 'I' in flag) ): hour = findHourFlag(flag) self.maybeTweetRecordSince(city, v, maxV, alreadyReportedRecord, hour, sampleDay, datemarker, maxRecord=True) if not (self.skipMinRecords or (self.skipMinEstimated and 'H' in flag) or (self.skipMinIncomplete and 'I' in flag) ): hour = findHourFlag(flag) self.maybeTweetRecordSince(city, v, minV, alreadyReportedRecord, hour, sampleDay, datemarker, maxRecord=False)
def main(city, dayToPlot, recently, thisDayInHistory, field): dailyData = daily.load(city) dailyDataClean(dailyData) if dayToPlot is None: dayToPlot = datetime.date.today() if recently: historyLines = daysRecently(dailyData, dayToPlot, field) else: historyLines = daysToday(dailyData, dayToPlot, field) if len(historyLines) == 1: #Most ever startDate = dailyData.firstDateWithValue(field.index) else: startDate = historyLines[1].date # - datetime.timedelta(days=(historyLines[0].date - historyLines[1].date).days * .1) endDate = historyLines[0].date + datetime.timedelta(days=1) plotData = [] for date in daily.dayRange(startDate, endDate): values = dailyData.get(date, None) if values is None: continue plotData.append( (date, values[field.index]) ) style=pygal.style.Style(label_font_size=15, major_label_font_size=20) date_chart = pygal.DateLine(style=style, print_values=True, #truncate_label=1000, x_label_rotation=80) date_chart.y_title = '{} ({})'.format(field.englishName, field.units) if field.minValue == 0: date_chart.add(None, plotData, show_dots=False, fill=True) else: date_chart.add(None, plotData, show_dots=False) labelByValue = {} for history in historyLines[0:2]: labelByValue[float(history.val)] = history.label date_chart.add(None, ( ( startDate, history.val ), ( historyLines[0].date, history.val ) ), formatter = lambda t: labelByValue[t[1]] ) historyType = '' if thisDayInHistory: historyType = 'thisDayInHistory' if recently: historyType = 'recently' fname = '{city}/{dayToPlot}.recordSince.{historyType}.png'.format(**locals()) date_chart.render_to_png(fname, width=1024, height=768) return fname
def yearlyValues(cityData, year, field): values = [] for date in daily.dayRange(datetime.date(year, 1, 1), datetime.date(year + 1, 1, 1)): #for date in daily.dayRange(datetime.date(year,6,1), datetime.date(year,9,1)): try: val = field(cityData[date]) if val != None: values.append(val) except KeyError: pass return values
def dailyDataClean(data): for year in range(data.minYear, data.maxYear+1): for day in daily.dayRange( datetime.date(year, 7, 1), datetime.date(year, 1, 1), -1): if day not in data: continue vals = data[day] if vals.SNOW_ON_GRND_CM is not None and vals.SNOW_ON_GRND_CM > 0: break vals = vals._replace(SNOW_ON_GRND_CM = 0) #print('Replacing missing data {day} with zero'.format(**locals())) data[day] = vals for day in daily.dayRange( datetime.date(year, 7, 1), datetime.date(year, 12, 31), 1): if day not in data: continue vals = data[day] if vals.SNOW_ON_GRND_CM is not None and vals.SNOW_ON_GRND_CM > 0: break vals = vals._replace(SNOW_ON_GRND_CM = 0) #print('Replacing missing data {day} with zero'.format(**locals())) data[day] = vals
def __call__(self, year): #import pdb; pdb.set_trace() if type(year) != tuple: year = (year, ) for y in year: baseDate = datetime.date(y, self.month, 1) endYear = y endMonth = self.month + 1 if endMonth > 12: endMonth = 1 endYear += 1 endDate = datetime.date(endYear, endMonth, 1) yield from daily.dayRange(baseDate, endDate)
def annualData(rawData, field, plotDate): dataByYear = {} # for year in range(rawData.minYear, rawData.maxYear + 1): # nextMonthYear = year nextMonthMonth = plotDate.month if nextMonthMonth > 11: nextMonthMonth = 1 nextMonthYear += 1 for day in daily.dayRange( datetime.date(year, plotDate.month, 1), datetime.date(nextMonthYear, nextMonthMonth, 1)): if day in rawData: if len(rawData[day][field.index]) > 0: dataByYear[day] = Fraction(rawData[day][field.index]) return dataByYear
def yearlySum(cityData, year, field): sum = 0 count = 0 fakeCount = 0 for date in daily.dayRange(START_DATE(year), END_DATE(year)): #print date try: val = field(cityData[date]) if val != None: sum += val count += 1 elif cityData[date].MAX_TEMP is not None: fakeCount += 1 except KeyError: pass if count > 30: count += fakeCount return (sum, count)
def yearlySum(cityData, year, field): sum = 0 count = 0 fakeCount = 0 for date in daily.dayRange(datetime.date(year, 1, 1), datetime.date(year + 1, 1, 1)): try: val = field(cityData[date]) if val != None: sum += val count += 1 elif len(cityData[date].MAX_TEMP) > 0: fakeCount += 1 except KeyError: pass if count > 30: count += fakeCount return (sum, count)
def rawdata(year, expr, dayFilter = None): ret = [] startDay = datetime.date(year, 1, 1) #January 1st endDay = datetime.date(year+1, 1, 1) #January 1st for day in daily.dayRange(startDay, endDay): if dayFilter != None and not dayFilter(day): continue try: dayValues = weatherData[day] except KeyError: continue result = expr(dayValues) if result != None: ret.append(result) return ret
def yearlySum(cityData, year, field): #print('yearlySum(', year, field.name, ')') sum = 0 count = 0 fakeCount = 0 for date in daily.dayRange(datetime.date(year, 1, 1), datetime.date(year + 1, 1, 1)): #for date in daily.dayRange(datetime.date(year,6,1), datetime.date(year,9,1)): try: val = field(cityData[date]) if val is not None: sum += val count += 1 elif cityData[date].MAX_TEMP is not None or cityData[ date].MIN_TEMP is not None: fakeCount += 1 except KeyError: pass #if count > 30: # count += fakeCount #assert(count>0) #print('yearlySum(', year, field.name, ')=', (sum, count)) return (sum, count)
def dayRange(self, city): startDate = today(city) - datetime.timedelta(days=6) return daily.dayRange(startDate, today(city) + datetime.timedelta(1))
#!/usr/bin/python3 # -*- coding: utf-8 -*- import daily import datetime field = daily.SNOW_ON_GRND_CM data = daily.load('ottawa') winterStartByYear = {} winterEndByYear = {} for year in range(data.minYear, data.maxYear + 1): daysCount = 0 missingDays = 0 days = daily.dayRange(datetime.date(year, 7, 1), datetime.date(year + 1, 7, 1)) for day in days: if day not in data or data[day].SNOW_ON_GRND_FLAG == 'M': if missingDays > 0: # Too many days missing, start from scratch daysCount = 0 missingDays = 0 else: missingDays += 1 daysCount += 1 continue if len(data[day].SNOW_ON_GRND_CM) > 0: depth = int(data[day].SNOW_ON_GRND_CM) if depth >= 1: daysCount += 1
def count(cityName, data, expr, name, endDate, minRunLen, showNth, verbose=True, skipIncomplete=False, ): runByStartDate = {} currentRunStartDate = None currentRun = [] todaysRun = 0 for sampleDate in daily.dayRange(min(data.keys()), endDate): dayValues = data.get(sampleDate, None) val = None flag = '' if dayValues != None: #print day, dayValues try: val = dayValues.eval(expr, {'time': sampleDate}) except AttributeError: print(dayValues) raise if val is not None: if ( val is False or ( type(val) is tuple and val[0] is False ) or skipIncomplete and 'I' in flag ): if len(currentRun) >= minRunLen: #print currentRunStartDate, len(currentRun), ['%.1f' % a for a in currentRun] runByStartDate[currentRunStartDate] = currentRun if sampleDate == endDate - dt.timedelta(1): print('today breaks a run of %d, starting on %s' % (len(currentRun), currentRunStartDate)) currentRun = [] currentRunStartDate = None else: if len(currentRun) == 0: currentRunStartDate = sampleDate if type(val) is tuple: currentRun.append(val[1]) else: currentRun.append(val) if sampleDate == endDate - dt.timedelta(1): todaysRun = len(currentRun) else: if len(currentRun) >= minRunLen: runByStartDate[currentRunStartDate] = currentRun currentRun = [] currentRunStartDate = None if len(currentRun) >= minRunLen: runByStartDate[currentRunStartDate] = currentRun #print runByStartDate startDateByRunLen = defaultdict(list) longestRunByWinter = defaultdict(int) for startDate, run in sorted(runByStartDate.items()): runlen = len(run) winter = winterFromDate(startDate) longestRunByWinter[winter] = max(runlen, longestRunByWinter[winter]) ds = dateDateStr( (startDate, startDate+dt.timedelta(days=runlen-1)) ) print(ds, len(run), run) #, startDateByRunLen startDateByRunLen[runlen] = list(reversed(sorted(startDateByRunLen[runlen]+[startDate]))) lengths = sorted(startDateByRunLen.keys()) most = lengths[-1] lastStartDate = max(runByStartDate.keys()) print('lastStartDate', lastStartDate) if verbose: print("most %s was %d (%s)" % (name, most, startDateByRunLen[most])) nth = 1 for l in reversed(lengths): for d in startDateByRunLen[l]: if nth <= showNth or d == lastStartDate: print('%d) %s (%d)' % (nth, dateDateStr( (d, d+dt.timedelta(days=l-1)) ), l), tuple(runByStartDate[d])) nth += len(startDateByRunLen[l]) print("today's run was %d" % todaysRun, currentRun) maxLengthPerModernWinter = tuple( longestRunByWinter[x] for x in filter( lambda t: t in range(winterFromDate(now)-30, winterFromDate(now)), longestRunByWinter.keys())) print('30-year average: {:.1f}' .format(sum(maxLengthPerModernWinter)/len(maxLengthPerModernWinter)))
def __call__(self, year): if type(year) != tuple: year = (year, ) for y in year: yield from daily.dayRange(datetime.date(y, self.month1, self.day1), datetime.date(y, self.month2, self.day2))
for a in x: std = std + (a - mean)**2 std = sqrt(std / float(n - 1)) return mean, std weatherData = daily.load('farm') rawData = {} for year in range(weatherData.minYear, weatherData.maxYear + 1): rawData[year] = [] startDay = datetime.date(year, 1, 1) endDay = datetime.date(year, 2, 1) for date in daily.dayRange(startDay, endDay): try: dayValues = weatherData[date] except KeyError: continue v = dayValues.MAX_TEMP if v != None and len(v) > 0: rawData[year].append(float(v)) yearBySwing = {} for year in sorted(rawData.keys()): d = rawData[year] if len(d) > 0: swings = []
def __call__(self, year): if type(year) is int: year = year, for y in year: yield from daily.dayRange(dt.date(y, 7, 1), dt.date(y + 1, 7, 1))
def first(cityName, expr, name, dateRange, excludeThisYear=False, order="first", yaxisLabel=None, run=1, limitToMostRecentYears=None, verboseIfInDateRange=None, ): data = daily.load(cityName) if limitToMostRecentYears != None: ndata = daily.Data() yearsAgo = datetime.date(now.year-limitToMostRecentYears, now.month, now.day) for key in data: if key >= yearsAgo: ndata[key] = data[key] data = ndata fieldDict = { 'min': Value(daily.MIN_TEMP), 'max': Value(daily.MAX_TEMP), 'tempSpan': ValueDiff(daily.MAX_TEMP, daily.MIN_TEMP), 'rain': Value(daily.TOTAL_RAIN_MM), 'humidex': Value(daily.MAX_HUMIDEX), 'snow': Value(daily.TOTAL_SNOW_CM), 'snowpack': ValueEmptyZero(daily.SNOW_ON_GRND_CM), 'windgust': Value(daily.SPD_OF_MAX_GUST_KPH), 'wind': Value(daily.AVG_WIND), 'windchill': Value(daily.MIN_WINDCHILL), 'avgWindchill': Value(daily.MIN_WINDCHILL), } fieldValues = fieldDict.values() referencedValues=[] for fieldName in fieldDict.keys(): if '{'+fieldName+'}' in expr: referencedValues.append(fieldName) firstByYear = {} eThisYear = now.year endTimeThisYear = datetime.date(now.year, dateRange.endMonth, dateRange.endDay) if dateRange.yearCross() and now < endTimeThisYear: eThisYear -= 1 for baseyear in range(data.minYear, data.maxYear-dateRange.todayIsYearCross()+1): try: dayOffset = datacache.readCache(cityName, baseyear, '{}.{}.{}.{}'.format(name,order,str(dateRange),expr)) if dayOffset != None: firstByYear[baseyear] = dayOffset continue except datacache.NotInCache: pass starttime = datetime.date(baseyear, dateRange.startMonth, dateRange.startDay) endtime = datetime.date(baseyear+dateRange.yearCross(), dateRange.endMonth, dateRange.endDay) #print baseyear, starttime, endtime dayRange = daily.dayRange(starttime,endtime) if order=="last": dayRange = daily.dayRange(endtime-datetime.timedelta(1), starttime-datetime.timedelta(1), -1) if excludeThisYear and baseyear == now.year-dateRange.yearCross(): # don't consider this year when looking for records for last event, because this year is not over continue expectedDayCount = 0 observedDayCount = 0 for day in dayRange: expectedDayCount += 1 if day in data: observedDayCount += 1 else: continue if baseyear in firstByYear: # We already figured out which day was first continue vals = {} for fieldName, fieldCall in fieldDict.items(): try: vals[fieldName] = fieldCall(data[day], day) vals[fieldName+'Flag'] = '"' + fieldCall.getFlag(data[day]) + '"' except KeyError: #print day, 'KeyError' vals[fieldName] = None skip=False usedVals={} for fieldName in fieldDict.keys(): if fieldName in referencedValues: if vals[fieldName] is None: #print 'Skipping {} because {} is None'.format(day, fieldName) skip=True break usedVals[fieldName] = vals[fieldName] if skip: continue #resolvedExpr = expr.format(**vals) #print(vals) #val = eval(expr) val = eval(expr, vals) #if True: #day == datetime.date(2015,10,17): #print day, resolvedExpr, val if val is True: dayOffset = (day - starttime).days firstByYear[baseyear] = dayOffset break observedPercent = observedDayCount * 100 / expectedDayCount if observedPercent < 85 and baseyear != eThisYear: print('Skipping {baseyear} because it only had {observedPercent:.1f}% of the days'.format(**locals())) if baseyear in firstByYear: firstByYear.pop(baseyear) elif baseyear not in firstByYear and baseyear != eThisYear: print('Event did not happen during {}.'.format(baseyear)) datacache.cacheThis(cityName, baseyear, '{}.{}.{}.{}'.format(name,order,str(dateRange),expr), firstByYear.get(baseyear, None)) yearByFirst = reverseDict(firstByYear) #for offset in sorted(yearByFirst.keys()): # for year in yearByFirst[offset]: # print datetime.date(year, dateRange.startMonth, dateRange.startDay) + datetime.timedelta(offset) #print yearByFirst verbose = False if verboseIfInDateRange == None: verbose = True elif eThisYear in firstByYear: thisYearFirstDayOffset = firstByYear[eThisYear] firstThisYearDate = ( datetime.date(eThisYear, dateRange.startMonth, dateRange.startDay) + datetime.timedelta(thisYearFirstDayOffset) ) if ( firstThisYearDate >= verboseIfInDateRange[0] and firstThisYearDate <= verboseIfInDateRange[1] ): verbose = True (earliest, secondEarliest, *ignore) = sorted(filter(lambda y: y!=now.year-dateRange.yearCross(), yearByFirst.keys()))[:2] + [None,None] (secondLatest, latest, *ignore) = sorted(filter(lambda y: y!=now.year-dateRange.yearCross(), yearByFirst.keys()))[-2:] + [None,None] earliestYears = lookupEmptyListIfNone(yearByFirst, earliest) secondEarliestYears = lookupEmptyListIfNone(yearByFirst, secondEarliest) latestYears = lookupEmptyListIfNone(yearByFirst, latest) secondLatestYears = lookupEmptyListIfNone(yearByFirst, secondLatest) earliestDates=[] latestDates=[] if verbose: earliestDates=showRecords(name, 'earliest', earliestYears, firstByYear, dateRange) showRecords(name, '2nd earliest', secondEarliestYears, firstByYear, dateRange) showRecords(name, '2nd latest', secondLatestYears, firstByYear, dateRange) latestDates=showRecords(name, 'latest', latestYears, firstByYear, dateRange) avgKeys = filter(lambda t: t in firstByYear, range(eThisYear-30, eThisYear)) offsets = sorted([firstByYear[a] for a in avgKeys]) avg = calcAvg(offsets) median = calcMedian(offsets) avgFirst = calcDate(eThisYear, dateRange.startMonth, dateRange.startDay, avg) medianFirst = calcDate(eThisYear, dateRange.startMonth, dateRange.startDay, median) if verbose: print("average {name} is {avgFirst}".format(**locals())) print("median {name} is {medianFirst}".format(**locals())) ret = None if eThisYear in firstByYear: thisYearFirst = firstByYear[eThisYear] countEarlier = 0 countEqual = 0 countLater = 0 recentCountEarlier = [] recentCountEqual = [] recentCountLater = [] for first in sorted(yearByFirst.keys()): if first < thisYearFirst: countEarlier += len(yearByFirst[first]) recentCountEarlier += filter(lambda y: y<eThisYear and y > eThisYear-31, yearByFirst[first]) elif first == thisYearFirst: countEqual += len(yearByFirst[first]) - 1 #Subtract the current year recentCountEqual += filter(lambda y: y<eThisYear and y > eThisYear-31, yearByFirst[first]) else: countLater += len(yearByFirst[first]) recentCountLater += filter(lambda y: y<eThisYear and y > eThisYear-31, yearByFirst[first]) totalCount = countEarlier + countEqual + countLater totalRecentCount = len(recentCountEarlier) + len(recentCountEqual) + len(recentCountLater) firstThisYear = ( datetime.date(eThisYear, dateRange.startMonth, dateRange.startDay) + datetime.timedelta(thisYearFirst) ) if verbose: print('%s %s was %s' % (now.year, name, firstThisYear )) print('%d%% of last %d years were earlier.' % (round(countEarlier * 100.0 / totalCount), totalCount)) print('%d%% of last %d years were the same.' % (round(countEqual * 100.0 / totalCount), totalCount)) print('%d%% of last %d years were later.' % (round(countLater * 100.0 / totalCount), totalCount)) print('%d of last %d years were earlier.' % (len(recentCountEarlier), totalRecentCount), sorted(recentCountEarlier)) print('%d of last %d years were the same.' % (len(recentCountEqual), totalRecentCount), sorted(recentCountEqual)) print('%d of last %d years were later.' % (len(recentCountLater), totalRecentCount), sorted(recentCountLater)) ret = firstThisYear, medianFirst, earliestDates, latestDates else: print('Not showing this year because eThisYear="{}" and firstByYear="{}"'.format(eThisYear, firstByYear)) if len(yearByFirst) == 0: # There's nothing to plot, so don't even bother trying return ret plotDataOtherYears = [] plotDataThisYear = [] dateLabels = [] for key in sorted(yearByFirst.keys()): if eThisYear in yearByFirst[key]: plotDataThisYear.append((key, len(yearByFirst[key]), '#%s' % ','.join(map(str, yearByFirst[key])) )) else: plotDataOtherYears.append((key, len(yearByFirst[key]), '#%s' % ','.join(map(str, yearByFirst[key])))) histogramFname = '%s/svg/%s' % (cityName, name.replace(' ', '_')) if ret != None: ret = ret + (histogramFname,) plot = gnuplot.Plot(histogramFname) dateMin = min(yearByFirst.keys()) dateMax = max(yearByFirst.keys()) plotDateMin = dateMin-1 #int(dateMin - (dateMax - dateMin)*.25) plotDateMax = dateMax+1 #int(dateMax + (dateMax - dateMin)*.25) for dayOffset in range(plotDateMin, plotDateMax+1, 1): date = datetime.date(now.year, dateRange.startMonth, dateRange.startDay) + datetime.timedelta(dayOffset) if date.day % 5 == 1 and date.day != 31: dateLabels.append('"%s" %d' % (date.strftime('%b/%d'), dayOffset)) ylabel = 'Number of years when %s happened on this day' % name if yaxisLabel != None: ylabel = yaxisLabel plot.open(title='%s in %s' % (name, cityName.capitalize()), ylabel=ylabel, xmin=plotDateMin, xmax=plotDateMax, ymin=0, ymax=max(len(a) for a in yearByFirst.values())+1, xtics=dateLabels, xticsRotate=45, margins=[7,8,2,5]) plot.addLine(gnuplot.Line('Other years', plotDataOtherYears, plot='boxes')) plot.addLine(gnuplot.Line('This year', plotDataThisYear, plot='boxes')) plot.plot() plot.close() print('') plotdata = [] for (year, plotCount) in firstByYear.items(): if plotCount != None: plotdata.append( (year, plotCount, ( '#' +str(datetime.date(year, dateRange.startMonth, dateRange.startDay) + datetime.timedelta(plotCount))))) ytics = [] for m in range(1,13): for d in (1,10,20): thisDate = datetime.date(2015,m,d) dayOffset = (thisDate - datetime.date(2015, dateRange.startMonth, dateRange.startDay)).days ytics.append('"%s" %d' % (thisDate.strftime('%b %d'), dayOffset)) #print(tuple(t[:2] for t in plotdata)) lineFit = linear.linearTrend(tuple(t[:2] for t in plotdata)) plot = gnuplot.Plot("%s/svg/%s_%s-%u_%s.line" % (cityName, name.replace(' ', '_'), monthName(dateRange.startMonth), dateRange.startDay, dateRange.lastDay().replace(' ', '-') ), yaxis=2) plot.open(title='%s %s between %s %u and %s' % (cityName.capitalize(), name, monthName(dateRange.startMonth), dateRange.startDay, dateRange.lastDay() ), ytics=ytics) plot.addLine(gnuplot.Line("Linear", lineFit, lineColour='green', plot='lines')) plot.addLine(gnuplot.Line("Date", plotdata, lineColour='purple')) plot.plot() plot.close() return ret
def __call__(self, city): if self.field.name in stations.city[city].skipDailyFields: # This city doesn't keep track of this information in a useful way. return ret = [] startDate = today(city) - datetime.timedelta(days=7) lyear = 0 for sampleDay in daily.dayRange(startDate, today(city)): if sampleDay.year != lyear: monthlyRecords = dailyRecords.getMonthRecords( city, sampleDay.year, self.field) lyear = sampleDay.year datemarker = dateMarkerFromDate(sampleDay) try: v = citydata[sampleDay][self.field.index] if v != None and self.datemarker.get(datemarker, None) != v: maxr = monthlyRecords.max[sampleDay.month - 1] minr = monthlyRecords.min[sampleDay.month - 1] msg = '' if v > maxr.value and self.datemarker.get( datemarker, None) != v: print({ 'date': sampleDay, 'month': monthName(sampleDay.month, long=True), 'field': self.field.name, 'value': v, 'units': self.field.htmlunits, 'recordValue': maxr.value, 'recordYear': maxr.year, 'recordMonth': maxr.month, 'recordDay': maxr.day }) tweet = ( '{date}: {month} record high {field} of {value:.1f}{units} breaks previous record of {recordValue:.1f}{units} from {recordYear}/{recordMonth:02d}/{recordDay:02d}\n' .format(date=sampleDay, month=monthName(sampleDay.month, long=True), field=self.field.name, value=float(v), units=self.field.htmlunits, recordValue=float(maxr.value), recordYear=maxr.year, recordMonth=maxr.month, recordDay=maxr.day)) msg += tweet self.datemarker[datemarker] = v, tweet elif v < minr.value and self.skipMinRecords is False: print({ 'date': sampleDay, 'month': monthName(sampleDay.month, long=True), 'field': self.field.name, 'value': v, 'units': self.field.htmlunits, 'recordValue': minr.value, 'recordYear': minr.year, 'recordMonth': minr.month, 'recordDay': minr.day }) tweet = ( '{date}: {month} record low {field} of {value:.1f}{units} breaks previous record of {recordValue:.1f}{units} from {recordYear}/{recordMonth:02d}/{recordDay:02d}\n' .format(date=sampleDay, month=monthName(sampleDay.month, long=True), field=self.field.name, value=float(v), units=self.field.htmlunits, recordValue=float(minr.value), recordYear=minr.year, recordMonth=minr.month, recordDay=minr.day)) msg += tweet self.datemarker[datemarker] = v, tweet if len(msg) > 0: ret.append( Email(field=self.field, date=sampleDay, message=msg)) except KeyError: pass return ret
def __call__(self, city): startDate = today(city) - datetime.timedelta(days=7) for sampleDay in daily.dayRange(startDate, today(city) + datetime.timedelta(1)): endDay = sampleDay + datetime.timedelta(1) startMonth, startDay = self.startMonthDay(sampleDay.month, sampleDay.day) (svgFname, result, mostSince, prevRecord, prevRecordYears) = (first.dispatch( city, firstYear=None, startMonth=startMonth, startDay=startDay, endMonth=endDay.month, endDay=endDay.day, expression=self.expr, verbose=False, skipIncomplete=self.skipIncomplete)) datemarker = dateMarkerFromDate(sampleDay) alreadyReportedRecord = self.datemarker.get(datemarker, 0) if result <= alreadyReportedRecord: try: os.unlink(svgFname + '.svg') except OSError as err: if err.errno != 2: raise else: dayString = 'Today is' if sampleDay != today(city): dayString = '%s %d was' % (monthName( sampleDay.month), sampleDay.day) context = '??' if mostSince != None: hashtag = '' if (mostSince <= startDate.year - 20 and (today(city) - sampleDay).days < 2): hashtag = { 'ottawa': ' #OttNews', 'toronto': ' #ToNews' }[city] context = 'Most since %d.%s' % (mostSince, hashtag) elif len(prevRecordYears) > 0: context = ( 'Broke previous record of %d in %s. #%s' % (prevRecord, '/'.join(map(str, prevRecordYears)), { 'ottawa': 'OttNews', 'toronto': 'ToNews' }[city])) tweet = ( '%s #%s\'s %s day with %s during %s. %s' % (dayString, stations.city[city].name, nth(result), self.label, self.timeBinDescription(sampleDay), context)) if len(tweet) > 117: tweet = tweet.replace(' below ', ' < ') print(tweet) if (mostSince == None and len(prevRecordYears) > 0 ) or mostSince <= startDate.year - 10: pngname = svgFname.replace('/svg/', '/') + '.png' command = 'rsvg-convert -o %s --background-color=white %s.svg' % ( pngname, svgFname) #print command assert os.system(command) == 0 #print 'api.PostMedia(%s, %s)' % (repr(tweet), repr(pngname)) if auto: response = 'y' else: response = input('"n" to cancel this tweet: ').strip() if response == 'y': delayedTweets.addToEndOfListForCity( city, tweet, pngname) if response != 'c': self.datemarker[datemarker] = result else: self.datemarker[datemarker] = result os.unlink(svgFname + '.svg') if (city == 'ottawa' and ((mostSince == None and len(prevRecordYears) > 0) or mostSince <= startDate.year - 20)): sendMail.sendMailConsole("*****@*****.**", alert.address, subject=alert.title, text=tweet, files=[pngname], auto=auto)