def loadHourlyData(city, dayField, hourField, day): if dayField == daily.TOTAL_SNOW_CM: return metarParse.loadSnowWithHours(city) elif dayField in (daily.MAX_TEMP, daily.MIN_TEMP): return loadHourlyDataSwob(city, dayField, day) mytimezone = stations.city[city].timezone cityData = daily.load(city) utcDayStart = datetime.datetime(day.year, day.month, day.day, 6, tzinfo=datetime.timezone.utc) dayStart = utcDayStart.astimezone(mytimezone) dayEnd = dayStart + datetime.timedelta(days=1) cityHourData = hourly.load(city, dateRange=(dayStart, dayEnd)) valByHour = {} for utcHour, values in cityHourData.items(): v = hourField(values) if v is None: continue l = utcHour.astimezone(mytimezone) valByHour[l] = v return { day: metarParse.SnowAndHourly(cityData[day][dayField.index], valByHour) }
def getValuesByYear(cityName, field, cumulative): cityData = daily.load(cityName) cityYearValues = annualAverages.getAnnualValues(cityData, field, cumulative) cityValuesByYear = {} for year, value in cityYearValues: cityValuesByYear[year] = value return cityValuesByYear
def main(city, dayToPlot, recently, thisDayInHistory, field): dailyData = daily.load(city) dailyDataClean(dailyData) if dayToPlot is None: dayToPlot = datetime.date.today() if recently: historyLines = daysRecently(dailyData, dayToPlot, field) else: historyLines = daysToday(dailyData, dayToPlot, field) if len(historyLines) == 1: #Most ever startDate = dailyData.firstDateWithValue(field.index) else: startDate = historyLines[1].date # - datetime.timedelta(days=(historyLines[0].date - historyLines[1].date).days * .1) endDate = historyLines[0].date + datetime.timedelta(days=1) plotData = [] for date in daily.dayRange(startDate, endDate): values = dailyData.get(date, None) if values is None: continue plotData.append( (date, values[field.index]) ) style=pygal.style.Style(label_font_size=15, major_label_font_size=20) date_chart = pygal.DateLine(style=style, print_values=True, #truncate_label=1000, x_label_rotation=80) date_chart.y_title = '{} ({})'.format(field.englishName, field.units) if field.minValue == 0: date_chart.add(None, plotData, show_dots=False, fill=True) else: date_chart.add(None, plotData, show_dots=False) labelByValue = {} for history in historyLines[0:2]: labelByValue[float(history.val)] = history.label date_chart.add(None, ( ( startDate, history.val ), ( historyLines[0].date, history.val ) ), formatter = lambda t: labelByValue[t[1]] ) historyType = '' if thisDayInHistory: historyType = 'thisDayInHistory' if recently: historyType = 'recently' fname = '{city}/{dayToPlot}.recordSince.{historyType}.png'.format(**locals()) date_chart.render_to_png(fname, width=1024, height=768) return fname
def dispatch(cityName, firstYear, endDate, expression, minRunLen, showNth, verbose=True, skipIncomplete=False): data = daily.load(cityName) if firstYear != None: for key in tuple(data.keys()): if key.year < firstYear: del data[key] return count(cityName, data, expression, '??', endDate, minRunLen, showNth, verbose, skipIncomplete=skipIncomplete)
def loadHourlyDataSwob(city, dayField, day): if dayField == daily.TOTAL_SNOW_CM: return metarParse.loadSnowWithHours(city) mytimezone = stations.city[city].timezone cityData = daily.load(city) utcDayStart = datetime.datetime(day.year, day.month, day.day, 6, tzinfo=datetime.timezone.utc) dayStart = utcDayStart.astimezone(mytimezone) dayEnd = dayStart + datetime.timedelta(days=1) cityHourData = gatherSwob.parse(city) hourly.load(city, dateRange=(dayStart, dayEnd)) valByHour = {} for valuesDict in cityHourData: ts = valuesDict['time'] if ts < dayStart or ts >= dayEnd: continue if ts == utcDayStart: v = valuesDict['air_temp'] elif dayField == daily.MAX_TEMP: v = valuesDict['max_air_temp_pst1hr'] elif dayField == daily.MIN_TEMP: v = valuesDict['min_air_temp_pst1hr'] else: assert (False) if v is None: continue v = D(v) l = ts.astimezone(mytimezone) valByHour[l] = v dayData = cityData.get(day, None) if dayData is not None: return { day: metarParse.SnowAndHourly(dayData[dayField.index], valByHour) } return {day: metarParse.SnowAndHourly(None, valByHour)}
def sinceWhen(city, valueIndex, recordDate): cityData = daily.load(city) if recordDate is None: firstDateWithObservation = cityData.firstDateWithValue(valueIndex) sinceWhat = ( 'since records began in {begin}' .format(begin=firstDateWithObservation.year) ) return sinceWhat sinceWhat = 'since {}, {}'.format(dayDescription(city, recordDate), recordDate.year) ageYears = yearsDiff(today(city), recordDate) if ageYears < 1: sinceWhat = 'since {}'.format(dayDescription(city, recordDate)) if ageYears > 2: sinceWhat = ( "in more than {} years, since {}, {}" .format(int(ageYears), dayDescription(city, recordDate), recordDate.year) ) if ageYears - int(ageYears) > 0.330: sinceWhat = ( "in almost {} years, since {}, {}" .format(int(ageYears+1), dayDescription(city, recordDate), recordDate.year) ) return sinceWhat
def main(city, year): data = daily.load(city) cold = ValWithDate(+999, None) heat = ValWithDate(-999, None) snow = ValWithDate(0, None) rain = ValWithDate(0, None) wind = ValWithDate(0, None) for day, weather in data.items(): if day.year != year: continue #import pudb; pu.db if (weather.MAX_TEMP is not None and weather.MAX_TEMP > heat.val): heat = ValWithDate(weather.MAX_TEMP, day) if (weather.MIN_TEMP is not None and weather.MIN_TEMP < cold.val): cold = ValWithDate(weather.MIN_TEMP, day) if (weather.TOTAL_SNOW_CM is not None and weather.TOTAL_SNOW_CM > snow.val): snow = ValWithDate(weather.TOTAL_SNOW_CM, day) if (weather.TOTAL_RAIN_MM is not None and weather.TOTAL_RAIN_MM > rain.val): rain = ValWithDate(weather.TOTAL_RAIN_MM, day) if (weather.SPD_OF_MAX_GUST_KPH is not None and weather.SPD_OF_MAX_GUST_KPH > wind.val): wind = ValWithDate(weather.SPD_OF_MAX_GUST_KPH, day) cityName = stations.city[city].name tweet = '#{cityName} 2016 extremes:'.format(**locals()) for name, u in (('heat', '℃'), ('cold', '℃'), ('snow', 'cm'), ('rain', 'mm'), ('wind', 'km/h')): if locals()[name].date is None: continue tweet += ('\n{}: {}{} ({})'.format( name, locals()[name].val, u, locals()[name].date.strftime('%b %d'))) alertTweets.maybeTweetWithSuffix(city, tweet)
#!/usr/bin/python3 import daily, time import argparse parser = argparse.ArgumentParser( description='Calculate monthly average and charts.') parser.add_argument('--city', default='ottawa') parser.add_argument('--span') args = parser.parse_args() data = daily.load(args.city) span = int(args.span) days = [] maxSwing = 0 for date in reversed(sorted(data.keys())): values = data[date] if len(values.MAX_TEMP) > 0 and len(values.MIN_TEMP) > 0: val = float(values.MIN_TEMP) days.append(val) val = float(values.MAX_TEMP) days.append(val) while len(days) > span * 2: days.pop(0) swing = max(days) - min(days) if swing > maxSwing: maxSwing = swing print(date, swing, days)
def main(city, args, lastCheckedValue=None, today=None): #import pdb; pdb.set_trace() monthlyAverages.cityData = daily.load(city) if today is None: today = daily.timeByCity[city].date() monthDate = today - datetime.timedelta(7) nextMonthStart = datetime.date( monthDate.year if monthDate.month < 12 else monthDate.year + 1, monthDate.month + 1 if monthDate.month < 12 else 1, 1) daysLeft = (nextMonthStart - today).days monthlyAverages.now = monthDate tomorrow = today + datetime.timedelta(days=1) fieldList = [] if args.daysAbove: fieldList += [(ExprVal('max>=' + str(t), title=str(t) + "℃", name="max>=" + str(t), units="days", unit="day", precision=0, field=daily.MAX_TEMP, description=str(t) + "℃ days"), True) for t in range(20, 29 + 1)] if args.daysBelow: fieldList += [(ExprVal('min<=' + str(t), title=str(t) + "℃", name="min<=" + str(t), units="days", unit="day", precision=0, field=daily.MIN_TEMP, description=str(t) + "℃ nights"), True) for t in range(-10, 1)] #[ ( ExprVal('maxHumidex>max and maxHumidex>=' + str(t), # title=str(t) + " humidex", # name="humidex>=" + str(t), # units="days", # unit="day", # precision=0), # True ) for t in range(30, 51) ] if args.rain: if 'TOTAL_RAIN_MM' not in stations.city[args.city].skipDailyFields: fieldList += [(FractionVal(daily.TOTAL_RAIN_MM, "Rain"), True)] if args.snow: fieldList += [(FractionVal(daily.TOTAL_SNOW_CM, "snow"), True)] if args.snowDays: fieldList += ([(ExprVal('snow>0', title="snow", name="snow", units="days", unit="day", precision=0, field=daily.TOTAL_SNOW_CM, description="snow days"), True)] + [(ExprVal('snow>=' + str(t), title=str(t) + "cm snow", name="snow>=" + str(t), units="days", unit="day", precision=0, field=daily.TOTAL_SNOW_CM, description=str(t) + "cm snow days"), True) for t in range(5, 41, 5)]) if args.wind: fieldList += [(FractionVal(daily.AVG_WIND, "wind"), False)] if args.meanTemp: fieldList += [(ExprVal( 'meanTemp' ' if ("M" not in meanTempFlag and "I" not in meanTempFlag)' ' else ((max+min)/2' ' if ("M" not in minFlag and "I" not in minFlag' ' and "M" not in maxFlag and "I" not in maxFlag)' ' else None)', name="Mean temperature", title="Mean temperature", units="℃", unit="℃", precision=2, field=daily.MEAN_TEMP, description='avg. hourly temp'), False)] if args.avgTemp: fieldList += [(Avg(daily.MIN_TEMP, daily.MAX_TEMP, "Temperature"), False)] if args.humidity: fieldList += [(FractionVal(daily.MEAN_HUMIDITY, "Humidity"), False)] monthStr = monthName(monthDate.month, long=True) monFilter = monthlyAverages.MonthFilter(month=monthDate.month) nextMonthYear = (monthDate.year * 12 + monthDate.month) // 12 nextMonthMonth = monthDate.month % 12 + 1 #import pudb; pu.db for field, cumulative in fieldList: valByYear = monthlyAverages.yearDatas(monFilter, field=field, lastYear=monthDate.year, cumulative=cumulative) normalVal = monthlyAverages.normalMonthlyDatas(monthDate.year, monFilter, field, cumulative=cumulative) if monthDate.year not in valByYear: continue ci80 = confidenceInterval80(valByYear, monthDate) ci50 = confidenceInterval50(valByYear, monthDate) thisYearVal = valByYear[monthDate.year] maxSince = None minSince = None for year in reversed(sorted(valByYear.keys())): if year != monthDate.year: val = valByYear[year] if maxSince is None and val >= thisYearVal: maxSince = year if minSince is None and val <= thisYearVal: minSince = year del val maybeTweetTop5(city, valByYear, nextMonthYear, nextMonthMonth, monthStr, field, monthDate) maybeTweetMaxSince(city, valByYear, field, monthDate, monthStr, maxSince, cumulative) maybeTweetMinSince(city, valByYear, field, monthDate, monthStr, minSince, cumulative) insideCi80 = (thisYearVal >= ci80[0] and thisYearVal <= ci80[1]) insideCi50 = (thisYearVal >= ci50[0] and thisYearVal <= ci50[1]) amountDescription = '' if thisYearVal < ci50[0] and (field.units != 'days' or thisYearVal != 0): amountDescription = 'just ' if daysLeft > 5 and thisYearVal > ci50[0]: amountDescription = 'already ' amount = formatWithUnits(thisYearVal, field) if field.units == 'days': amount = formatWithUnits(thisYearVal, field, 0, skipUnits=True) if thisYearVal == 0: amount = "no" aboveBelow = 'above' if thisYearVal < normalVal.average: aboveBelow = 'below' if cumulative: deviation = deviationDescription(thisYearVal, normalVal.average) else: diff = abs(float(thisYearVal - normalVal.average)) deviation = formatWithUnits(diff, field) + ' ' + aboveBelow start = 'average' if cumulative: start = 'total' title = field.title.lower() ci80Low = formatWithUnits(ci80[0], field, skipUnits=True) ci80High = formatWithUnits(ci80[1], field) insideOutside = 'outside' if insideCi80: insideOutside = 'inside' cityName = stations.city[city].name average = formatWithUnits(normalVal.average, field, precision=max(field.precision, 1)) tweetText = ('#{cityName}\'s {start} {title} this {monthStr} was' ' {amountDescription}{amount},' ' {deviation} the average of {average}.'.format( **locals())) if daysLeft > 0: tweetText = ( f'With {daysLeft} days left, #{cityName}\'s {start} {title} this {monthStr} is' f' {amountDescription}{amount},' f' {deviation} the average of {average}.') if field.units == 'days': units = field.unit if thisYearVal != 1: units = field.units tweetText = ( '#{cityName} had' ' {amountDescription}{amount} {title} {units} this {monthStr},' ' {deviation} the average of {average}.'.format(**locals())) print(tweetText) #alertTweets.maybeTweetWithSuffix(city, tweetText) recordMin = normalVal.minimum recordMax = normalVal.maximum print( 'Record minimum was {recordMin}{field.units} in {normalVal.minYear}' .format(**locals())) print( 'Record maximum was {recordMax}{field.units} in {normalVal.maxYear}' .format(**locals())) plotIt = not insideCi80 or args.force if maxSince is None: print('***Max since records began in {}.'.format( min(valByYear.keys()))) plotIt = True elif monthDate.year - maxSince > 10: maxSinceVal = float(valByYear[maxSince]) print('***Max since {maxSince}:{maxSinceVal}{field.units}'.format( **locals())) plotIt = True if minSince is None: print('***Min since records began in {}.'.format( min(valByYear.keys()))) plotIt = True elif monthDate.year - minSince > 10: minSinceVal = float(valByYear[minSince]) print('***Min since {minSince}:{minSinceVal}{field.units}'.format( **locals())) plotIt = True if plotIt: monthlyAverages.annualOrderedByMonth(city, monFilter, field, cumulative=cumulative) plotFname = snow.createPlot( city, running=cumulative, field=field, otheryears=(), name=monthName(monthDate.month) + title.replace(' ', '_').replace('℃', 'C'), dataStartDay=datetime.date(monthDate.year, monthDate.month, 1), plotStartDay=datetime.date(monthDate.year, monthDate.month, 1), plotEndDay=datetime.date(nextMonthYear, nextMonthMonth, 1), plotYMin=None) pngname = plotFname.replace('/svg/', '/') + '.png' command = 'rsvg-convert -o {pngname} --background-color=white {plotFname}'.format( **locals()) print(command) assert os.system(command) == 0 alertTweets.maybeTweetWithSuffix(city, tweetText, fname=pngname)
def main(): HISTORY = 10 parser = argparse.ArgumentParser( description='Determine how often some weather occurs.') parser.add_argument('expr', help='Which observation to check') parser.add_argument('--city', default='ottawa') parser.add_argument('--run', type=int, default=1) parser.add_argument('-m', help='Mask', default=['*'], nargs='*') parser.add_argument( '--between', help= 'Only consider dates between these two. Comma separated like 05-15,07-01' ) parser.add_argument('--holiday', help='The name of a holiday') parser.add_argument('--hour', help='Use hourly data instead of daily.', action='store_true', default=False) parser.add_argument('--winters', help='Count by winter instead of by year.', action='store_true', default=False) parser.add_argument('--group-by-year', action='store_true', default=False) args = parser.parse_args() run = args.run city = args.city fieldDict = { 'min': Value(daily.MIN_TEMP), 'max': Value(daily.MAX_TEMP), 'meanTemp': Value(daily.MEAN_TEMP), 'tempSpan': ValueDiff(daily.MAX_TEMP, daily.MIN_TEMP), 'rain': Value(daily.TOTAL_RAIN_MM), 'precip': Value(daily.TOTAL_PRECIP_MM), 'humidex': Value(daily.MAX_HUMIDEX), 'snow': Value(daily.TOTAL_SNOW_CM), 'snowpack': ValueEmptyZero(daily.SNOW_ON_GRND_CM), 'windgust': Value(daily.SPD_OF_MAX_GUST_KPH), 'wind': Value(daily.AVG_WIND), 'windchill': Value(daily.MIN_WINDCHILL), 'avgWindchill': Value(daily.MIN_WINDCHILL), } if args.hour: fieldDict = { 'temp': Value(hourly.TEMP), 'dewpoint': Value(hourly.DEW_POINT_TEMP), 'humidity': Value(hourly.REL_HUM), 'windDir': Value(hourly.WIND_DIR), 'wind': Value(hourly.WIND_SPD), 'visibility': Value(hourly.VISIBILITY), 'pressure': Value(hourly.STN_PRESS), 'weather': ValueNoFlag(hourly.WEATHER), 'windchill': ValueNoFlag(hourly.WINDCHILL), } def allFieldNames(): return fieldDict.keys() dateFilter = args.m between = parseBetween(args.between) if args.hour: data = hourly.load(city) for time, conditions in parseWeatherStatsRealtime.parse(city).items(): if time.minute == 0: values = data.get(time, None) if values is None: data[time] = hourly.HourData(WEATHER=conditions) else: data[time] = values._replace(WEATHER=conditions) for time, metarWeather in metarParse.genHourlyWeather(city): if time.minute != 0: continue #if time.date() == dt.date(2017,3,27): # import pudb; pu.db values = data.get(time, None) if values is None: data[time] = hourly.HourData(WEATHER=metarWeather) elif (values.WEATHER is None or values.WEATHER == 'NA'): data[time] = values._replace(WEATHER=metarWeather) else: data = daily.load(city) maxVal = None curRun = deque() curDates = deque() fieldValues = fieldDict.values() firstDate = None matches = [] referencedValues = set() compiledExpr = compile(args.expr, filename='.', mode='eval') exprWords = pythonWords(args.expr) for fieldName in fieldDict.keys(): if fieldName in exprWords: referencedValues.add(fieldName) #print(referencedValues) #print(tuple(allFieldNames())) class refCountedList(deque): def __init__(self): self.indexSet = set() def __getitem__(self, ind): if type(ind) != slice: #print('[{ind}]'.format(**locals())) self.indexSet.add(ind) return deque.__getitem__(self, ind) return list(self)[ind] def clearIndexSet(self): self.indexSet.clear() historyDates = deque([]) history = refCountedList() class History(): pass expectedDiff = dt.timedelta(days=1) if args.hour: expectedDiff = dt.timedelta(hours=1) mytimezone = stations.city[args.city].timezone for date in sorted(data.keys()): if args.hour: utchour = date localhour = utchour.astimezone(mytimezone) date = localhour if (len(historyDates) > 0 and date - historyDates[0] != expectedDiff): historyDates.clear() history.clear() if matchDate(date, args.m, between, args.holiday): #if date.year == 2016 and date.month == 11 and date.day == 30: # import pudb; pu.db vals = { 'history': history, "__builtins__": __builtins__, 'time': date } history.appendleft(History()) historyDates.appendleft(date) for fieldName, fieldCall in fieldDict.items(): vals[fieldName] = fieldCall(data[date], date) flagValue = fieldCall.getFlag(data[date]) if flagValue is not None: vals[fieldName + 'Flag'] = flagValue if type(vals[fieldName]) is not SpecialNone: history[0].__setattr__(fieldName, vals[fieldName]) if flagValue is not None: history[0].__setattr__(fieldName + 'Flag', flagValue) skip = False usedVals = {} for fieldName in referencedValues: if type(vals[fieldName]) is SpecialNone: #print('Skipping {} because {} is None'.format(date, fieldName)) skip = True break #print(fieldName, type(vals[fieldName]) is SpecialNone) usedVals[fieldName] = vals[fieldName] if skip: continue if firstDate is None: firstDate = date lastDate = date #expr = args.expr.format(**vals) #print(date, args.expr, usedVals, vals) history.clearIndexSet() try: val = eval(compiledExpr, vals) except IndexError: val = False except AttributeError: val = False except TypeError: for offset, date in enumerate(historyDates): print(date) for name in allFieldNames(): if hasattr(history[offset], name): print(name, history[offset].__getattribute__(name)) raise #print(val) if val is True or type(val) is tuple and val[0] is True: #print('+++') #for offset, date in enumerate(historyDates): # print(date) # for name in allFieldNames(): # if name in dir(history[offset]): # print(name, history[offset].__getattribute__(name)) #print('---') for i in history.indexSet: for fieldName in referencedValues: usedVals['history[{}].{}'.format(i, fieldName)] = ( history[i].__getattribute__(fieldName)) #print(history.indexSet) expectedDate = date if len(curDates) > 0: expectedDate = curDates[-1] + expectedDiff if date != expectedDate: #print('Clearing run', date, curDates[-1]) curDates = deque([date]) curRun = deque([usedVals]) if type(val) is tuple: curRun = deque([{'expr': val[1]}]) else: curDates.append(date) if type(val) is tuple: curRun.append({'expr': val[1]}) else: curRun.append(usedVals) if len(curRun) > run: curRun.popleft() curDates.popleft() #print(date, len(curRun), run) if len(curRun) == run: printDate(curDates) if len(curRun[0]) == 1: print(' '.join( [str(first(a.values())) for a in curRun])) else: print(curRun) matches.append(curDates) if args.winters: grouping = tuple([winterFromDate(a[0]) for a in matches]) group = 'winter' else: grouping = tuple([a[0].year for a in matches]) group = 'year' if args.group_by_year: print(tuple(enumerate(Counter(grouping).most_common(30)))) print('Total count: {}'.format(len(grouping))) print('Occurance: ', end='') yearSpan = lastDate.year - firstDate.year + 1 if len(matches) < yearSpan: print('once every {:.1f} {}s'.format(yearSpan / len(matches), group)) else: print('{:.1f} times per {}'.format(len(matches) / yearSpan, group)) print('Yearly Occurance:', end=' ') uniqGrouping = sorted(list(set(grouping))) if len(uniqGrouping) < yearSpan: occurance = yearSpan / len(uniqGrouping) print('once every {occurance:.1f} {group}s,'.format(**locals()), end=' ') else: occurance = len(uniqGrouping) / yearSpan print('{occurance:.1f} times per {group},'.format(**locals()), end=' ') recentFilter = lambda t: t >= today().year - 30 and t < today().year if args.winters: recentFilter = lambda t: t >= thisWinter() - 30 and t < thisWinter() recentOccursYears = tuple(filter(recentFilter, grouping)) recentUniqYears = tuple(sorted(set(recentOccursYears))) print('{} out of the past 30 years: {}'.format(len(recentUniqYears), str(recentUniqYears))) recentYears = range(thisYear() - 30, thisYear()) if args.winters: recentYears = range(thisWinter() - 30, thisWinter()) recentLen = len(recentUniqYears) recentYearCounter = Counter(recentOccursYears) recentLens = sorted(recentYearCounter[a] for a in recentYears) print('{} during the past 30 years'.format(recentLen)) if True: #recentLen > 30: print('Average is {:.1f}/year during the past 30 years'.format( len(recentOccursYears) / 30)) print('Median is {},{}/year during the past 30 years'.format( recentLens[int(len(recentLens) / 2)], recentLens[int( (len(recentLens) + 1) / 2)])) print( '80% CI is {},{}'.format( recentLens[len(recentLens) // 10], recentLens[len(recentLens) - (len(recentLens) // 10) - 1]), recentLens) #print(sorted([(len(o),y) for y,o in recentByYear.items()])) else: print('Every {:.1f} years during the past 30 years'.format(30 / recentLen)) occurDays = '∞' if recentLen > 0: occurDays = (dt.date.today() - yearsAgo(30)).days / recentLen print('Every {} days during the past 30 years'.format(occurDays)) if recentLen > 0: print('Every {} months, {:.1f} days during the past 30 years'.format( int(occurDays // 30), occurDays % 30)) if args.hour: recentOccur = tuple( filter(lambda t: t[0].date() >= yearsAgo(1), matches)) else: recentOccur = tuple(filter(lambda t: t[0] >= yearsAgo(1), matches)) print('{} during the past year'.format(len(recentOccur))) if args.winters: thisWinterCount = grouping.count(thisWinter()) print('{} so far this winter'.format(thisWinterCount)) else: thisYearCount = grouping.count(thisYear()) print('{} so far this year'.format(thisYearCount))
'humidex': Value(daily.MAX_HUMIDEX), 'snow': Value(daily.TOTAL_SNOW_CM), 'snowpack': ValueEmptyZero(daily.SNOW_ON_GRND_CM), 'windgust': Value(daily.SPD_OF_MAX_GUST_KPH), 'wind': Value(daily.AVG_WIND), 'windchill': Value(daily.MIN_WINDCHILL), 'avgWindchill': Value(daily.AVG_WINDCHILL), 'meanHumidity': Value(daily.MEAN_HUMIDITY), 'minHumidity': Value(daily.MIN_HUMIDITY), 'minRec': ValueNearRecordMin(daily.MIN_TEMP), }[fieldName] dateFilter = args.m between = parseBetween(args.between) data = daily.load(city) maxVal = -999 curRun = collections.deque([-999] * run, run) curDates = collections.deque([datetime.date(1800, 1, 1)] * run, run) rawData = open('recordSince.csv', 'w') for date in reversed(sorted(data.keys())): if matchDate(date, [dateFilter], between, args.holiday): flag = field.getFlag(data[date]) if 'H' in flag and field.getName() in ('MIN_TEMP', 'MIN_WINDCHILL'): continue val = field(data[date], date) if val != None:
ret = MaxMin(dayInfo.max, dayInfo.min) maxMinCache[key] = ret return ret def getMonthRecords(city, targetYear, field): ret = MaxMin(max=[], min=[]) for i in range(12): ret.max.append(Record(-999, year=1900)) ret.min.append(Record(+999, year=1900)) cityData = daily.dataByCity[city] for date in sorted(cityData.keys()): if date.year == targetYear: break val = cityData[date][field.index] if val is not None: if val > ret.max[date.month - 1].value: ret.max[date.month - 1] = Record(val, date.year, date.month, date.day) if val < ret.min[date.month - 1].value: ret.min[date.month - 1] = Record(val, date.year, date.month, date.day) return ret if __name__ == '__main__': daily.load("hamilton") print(getInfo('hamilton', datetime.date(2017, 3, 1), daily.AVG_WIND))
from monthName import monthName now = datetime.date.today() - datetime.timedelta(3) cityName = 'ottawa' firstYear = None opts, args = getopt.getopt(sys.argv[1:], 'c:f:', ['city=', 'firstYear=']) for opt, arg in opts: if opt in ('-c','--city'): cityName = arg elif opt in ('-f','--firstYear'): firstYear = int(arg) cityData = daily.load(cityName) if firstYear != None: keys = cityData.keys() for key in keys: if key.year < firstYear: del cityData[key] class YearMonthFilter(): __slots__ = () def __init__(self, year, month): self.year = year self.month = month # def __call__(self, year = None): if year != None:
def first(cityName, expr, name, dateRange, excludeThisYear=False, order="first", yaxisLabel=None, run=1, limitToMostRecentYears=None, verboseIfInDateRange=None, ): data = daily.load(cityName) if limitToMostRecentYears != None: ndata = daily.Data() yearsAgo = datetime.date(now.year-limitToMostRecentYears, now.month, now.day) for key in data: if key >= yearsAgo: ndata[key] = data[key] data = ndata fieldDict = { 'min': Value(daily.MIN_TEMP), 'max': Value(daily.MAX_TEMP), 'tempSpan': ValueDiff(daily.MAX_TEMP, daily.MIN_TEMP), 'rain': Value(daily.TOTAL_RAIN_MM), 'humidex': Value(daily.MAX_HUMIDEX), 'snow': Value(daily.TOTAL_SNOW_CM), 'snowpack': ValueEmptyZero(daily.SNOW_ON_GRND_CM), 'windgust': Value(daily.SPD_OF_MAX_GUST_KPH), 'wind': Value(daily.AVG_WIND), 'windchill': Value(daily.MIN_WINDCHILL), 'avgWindchill': Value(daily.MIN_WINDCHILL), } fieldValues = fieldDict.values() referencedValues=[] for fieldName in fieldDict.keys(): if '{'+fieldName+'}' in expr: referencedValues.append(fieldName) firstByYear = {} eThisYear = now.year endTimeThisYear = datetime.date(now.year, dateRange.endMonth, dateRange.endDay) if dateRange.yearCross() and now < endTimeThisYear: eThisYear -= 1 for baseyear in range(data.minYear, data.maxYear-dateRange.todayIsYearCross()+1): try: dayOffset = datacache.readCache(cityName, baseyear, '{}.{}.{}.{}'.format(name,order,str(dateRange),expr)) if dayOffset != None: firstByYear[baseyear] = dayOffset continue except datacache.NotInCache: pass starttime = datetime.date(baseyear, dateRange.startMonth, dateRange.startDay) endtime = datetime.date(baseyear+dateRange.yearCross(), dateRange.endMonth, dateRange.endDay) #print baseyear, starttime, endtime dayRange = daily.dayRange(starttime,endtime) if order=="last": dayRange = daily.dayRange(endtime-datetime.timedelta(1), starttime-datetime.timedelta(1), -1) if excludeThisYear and baseyear == now.year-dateRange.yearCross(): # don't consider this year when looking for records for last event, because this year is not over continue expectedDayCount = 0 observedDayCount = 0 for day in dayRange: expectedDayCount += 1 if day in data: observedDayCount += 1 else: continue if baseyear in firstByYear: # We already figured out which day was first continue vals = {} for fieldName, fieldCall in fieldDict.items(): try: vals[fieldName] = fieldCall(data[day], day) vals[fieldName+'Flag'] = '"' + fieldCall.getFlag(data[day]) + '"' except KeyError: #print day, 'KeyError' vals[fieldName] = None skip=False usedVals={} for fieldName in fieldDict.keys(): if fieldName in referencedValues: if vals[fieldName] is None: #print 'Skipping {} because {} is None'.format(day, fieldName) skip=True break usedVals[fieldName] = vals[fieldName] if skip: continue #resolvedExpr = expr.format(**vals) #print(vals) #val = eval(expr) val = eval(expr, vals) #if True: #day == datetime.date(2015,10,17): #print day, resolvedExpr, val if val is True: dayOffset = (day - starttime).days firstByYear[baseyear] = dayOffset break observedPercent = observedDayCount * 100 / expectedDayCount if observedPercent < 85 and baseyear != eThisYear: print('Skipping {baseyear} because it only had {observedPercent:.1f}% of the days'.format(**locals())) if baseyear in firstByYear: firstByYear.pop(baseyear) elif baseyear not in firstByYear and baseyear != eThisYear: print('Event did not happen during {}.'.format(baseyear)) datacache.cacheThis(cityName, baseyear, '{}.{}.{}.{}'.format(name,order,str(dateRange),expr), firstByYear.get(baseyear, None)) yearByFirst = reverseDict(firstByYear) #for offset in sorted(yearByFirst.keys()): # for year in yearByFirst[offset]: # print datetime.date(year, dateRange.startMonth, dateRange.startDay) + datetime.timedelta(offset) #print yearByFirst verbose = False if verboseIfInDateRange == None: verbose = True elif eThisYear in firstByYear: thisYearFirstDayOffset = firstByYear[eThisYear] firstThisYearDate = ( datetime.date(eThisYear, dateRange.startMonth, dateRange.startDay) + datetime.timedelta(thisYearFirstDayOffset) ) if ( firstThisYearDate >= verboseIfInDateRange[0] and firstThisYearDate <= verboseIfInDateRange[1] ): verbose = True (earliest, secondEarliest, *ignore) = sorted(filter(lambda y: y!=now.year-dateRange.yearCross(), yearByFirst.keys()))[:2] + [None,None] (secondLatest, latest, *ignore) = sorted(filter(lambda y: y!=now.year-dateRange.yearCross(), yearByFirst.keys()))[-2:] + [None,None] earliestYears = lookupEmptyListIfNone(yearByFirst, earliest) secondEarliestYears = lookupEmptyListIfNone(yearByFirst, secondEarliest) latestYears = lookupEmptyListIfNone(yearByFirst, latest) secondLatestYears = lookupEmptyListIfNone(yearByFirst, secondLatest) earliestDates=[] latestDates=[] if verbose: earliestDates=showRecords(name, 'earliest', earliestYears, firstByYear, dateRange) showRecords(name, '2nd earliest', secondEarliestYears, firstByYear, dateRange) showRecords(name, '2nd latest', secondLatestYears, firstByYear, dateRange) latestDates=showRecords(name, 'latest', latestYears, firstByYear, dateRange) avgKeys = filter(lambda t: t in firstByYear, range(eThisYear-30, eThisYear)) offsets = sorted([firstByYear[a] for a in avgKeys]) avg = calcAvg(offsets) median = calcMedian(offsets) avgFirst = calcDate(eThisYear, dateRange.startMonth, dateRange.startDay, avg) medianFirst = calcDate(eThisYear, dateRange.startMonth, dateRange.startDay, median) if verbose: print("average {name} is {avgFirst}".format(**locals())) print("median {name} is {medianFirst}".format(**locals())) ret = None if eThisYear in firstByYear: thisYearFirst = firstByYear[eThisYear] countEarlier = 0 countEqual = 0 countLater = 0 recentCountEarlier = [] recentCountEqual = [] recentCountLater = [] for first in sorted(yearByFirst.keys()): if first < thisYearFirst: countEarlier += len(yearByFirst[first]) recentCountEarlier += filter(lambda y: y<eThisYear and y > eThisYear-31, yearByFirst[first]) elif first == thisYearFirst: countEqual += len(yearByFirst[first]) - 1 #Subtract the current year recentCountEqual += filter(lambda y: y<eThisYear and y > eThisYear-31, yearByFirst[first]) else: countLater += len(yearByFirst[first]) recentCountLater += filter(lambda y: y<eThisYear and y > eThisYear-31, yearByFirst[first]) totalCount = countEarlier + countEqual + countLater totalRecentCount = len(recentCountEarlier) + len(recentCountEqual) + len(recentCountLater) firstThisYear = ( datetime.date(eThisYear, dateRange.startMonth, dateRange.startDay) + datetime.timedelta(thisYearFirst) ) if verbose: print('%s %s was %s' % (now.year, name, firstThisYear )) print('%d%% of last %d years were earlier.' % (round(countEarlier * 100.0 / totalCount), totalCount)) print('%d%% of last %d years were the same.' % (round(countEqual * 100.0 / totalCount), totalCount)) print('%d%% of last %d years were later.' % (round(countLater * 100.0 / totalCount), totalCount)) print('%d of last %d years were earlier.' % (len(recentCountEarlier), totalRecentCount), sorted(recentCountEarlier)) print('%d of last %d years were the same.' % (len(recentCountEqual), totalRecentCount), sorted(recentCountEqual)) print('%d of last %d years were later.' % (len(recentCountLater), totalRecentCount), sorted(recentCountLater)) ret = firstThisYear, medianFirst, earliestDates, latestDates else: print('Not showing this year because eThisYear="{}" and firstByYear="{}"'.format(eThisYear, firstByYear)) if len(yearByFirst) == 0: # There's nothing to plot, so don't even bother trying return ret plotDataOtherYears = [] plotDataThisYear = [] dateLabels = [] for key in sorted(yearByFirst.keys()): if eThisYear in yearByFirst[key]: plotDataThisYear.append((key, len(yearByFirst[key]), '#%s' % ','.join(map(str, yearByFirst[key])) )) else: plotDataOtherYears.append((key, len(yearByFirst[key]), '#%s' % ','.join(map(str, yearByFirst[key])))) histogramFname = '%s/svg/%s' % (cityName, name.replace(' ', '_')) if ret != None: ret = ret + (histogramFname,) plot = gnuplot.Plot(histogramFname) dateMin = min(yearByFirst.keys()) dateMax = max(yearByFirst.keys()) plotDateMin = dateMin-1 #int(dateMin - (dateMax - dateMin)*.25) plotDateMax = dateMax+1 #int(dateMax + (dateMax - dateMin)*.25) for dayOffset in range(plotDateMin, plotDateMax+1, 1): date = datetime.date(now.year, dateRange.startMonth, dateRange.startDay) + datetime.timedelta(dayOffset) if date.day % 5 == 1 and date.day != 31: dateLabels.append('"%s" %d' % (date.strftime('%b/%d'), dayOffset)) ylabel = 'Number of years when %s happened on this day' % name if yaxisLabel != None: ylabel = yaxisLabel plot.open(title='%s in %s' % (name, cityName.capitalize()), ylabel=ylabel, xmin=plotDateMin, xmax=plotDateMax, ymin=0, ymax=max(len(a) for a in yearByFirst.values())+1, xtics=dateLabels, xticsRotate=45, margins=[7,8,2,5]) plot.addLine(gnuplot.Line('Other years', plotDataOtherYears, plot='boxes')) plot.addLine(gnuplot.Line('This year', plotDataThisYear, plot='boxes')) plot.plot() plot.close() print('') plotdata = [] for (year, plotCount) in firstByYear.items(): if plotCount != None: plotdata.append( (year, plotCount, ( '#' +str(datetime.date(year, dateRange.startMonth, dateRange.startDay) + datetime.timedelta(plotCount))))) ytics = [] for m in range(1,13): for d in (1,10,20): thisDate = datetime.date(2015,m,d) dayOffset = (thisDate - datetime.date(2015, dateRange.startMonth, dateRange.startDay)).days ytics.append('"%s" %d' % (thisDate.strftime('%b %d'), dayOffset)) #print(tuple(t[:2] for t in plotdata)) lineFit = linear.linearTrend(tuple(t[:2] for t in plotdata)) plot = gnuplot.Plot("%s/svg/%s_%s-%u_%s.line" % (cityName, name.replace(' ', '_'), monthName(dateRange.startMonth), dateRange.startDay, dateRange.lastDay().replace(' ', '-') ), yaxis=2) plot.open(title='%s %s between %s %u and %s' % (cityName.capitalize(), name, monthName(dateRange.startMonth), dateRange.startDay, dateRange.lastDay() ), ytics=ytics) plot.addLine(gnuplot.Line("Linear", lineFit, lineColour='green', plot='lines')) plot.addLine(gnuplot.Line("Date", plotdata, lineColour='purple')) plot.plot() plot.close() return ret
import daily, datetime def meanstdv(x): from math import sqrt n, mean, std = len(x), 0, 0 for a in x: mean = mean + a mean = mean / float(n) for a in x: std = std + (a - mean)**2 std = sqrt(std / float(n - 1)) return mean, std weatherData = daily.load('farm') rawData = {} for year in range(weatherData.minYear, weatherData.maxYear + 1): rawData[year] = [] startDay = datetime.date(year, 1, 1) endDay = datetime.date(year, 2, 1) for date in daily.dayRange(startDay, endDay): try: dayValues = weatherData[date] except KeyError: continue v = dayValues.MAX_TEMP
#!/usr/bin/python from __future__ import print_function import sys, datetime, daily, ps, forecast, sys, dailyRecords from collections import namedtuple city = sys.argv[1] historyDays = int(sys.argv[2]) now = forecast.now daily.load(city) Temp_MinMax = namedtuple('Temp_MinMax', 'min max') def getTempData(): history = Temp_MinMax({}, {}) for daysOffset in range(-historyDays, 8 + 1): date = now + datetime.timedelta(daysOffset) history.min[date] = dailyRecords.getInfo(city, date, daily.MIN_TEMP) history.max[date] = dailyRecords.getInfo(city, date, daily.MAX_TEMP) return history class COLDER(): __slots__ = () class WARMER():
#!/usr/bin/python # -*- coding: utf-8 -*- from __future__ import print_function import time, posix, daily data = daily.load("ottawa") class FloatValue(): __slots__ = () def __init__(self, field): self.fieldIndex = field.index def __call__(self, fields): r = fields[self.fieldIndex] if len(r) == 0: return None return float(r) class IntValue(): __slots__ = () def __init__(self, field): self.fieldIndex = field.index def __call__(self, fields): r = fields[self.fieldIndex] if len(r) == 0: return None
def main( city, force=False, lastCheckedValue=None, today=None, maxValueToCheck=None, allYear=False, allWinter=False, justTop5=False, dataSinceDay=None, ): #import pdb; pdb.set_trace() data = daily.load(city) if dataSinceDay is not None: for day in tuple(data.keys()): if day < dataSinceDay: del data[day] monthlyAverages.cityData = data if today is None: today = daily.timeByCity[city].date() #if allYear: # today = dt.date(daily.timeByCity[city].date().year, 12, 31) yearToCheck = (today - dt.timedelta(7)).year if yearToCheck != today.year: today = dt.date(yearToCheck, 12, 31) monthlyAverages.now = today tomorrow = today + dt.timedelta(days=1) todayMaxInfo = dailyRecords.getInfo(city, today, daily.MAX_TEMP) todayAverageMax = roundAwayFromZero(todayMaxInfo.normal) todayMax = None if todayMaxInfo.recent is not None: todayMax = int(todayMaxInfo.recent) if lastCheckedValue is None: minValueToCheck = int(todayAverageMax) + 2 else: minValueToCheck = int(lastCheckedValue) + 1 if maxValueToCheck is None: maxValueToCheck = todayMax if todayMax is None: maxValueToCheck = 35 maxValuesToCheck = filter(lambda t: t % 10 == 0, range(minValueToCheck, maxValueToCheck + 1)) fieldList = [ #*[ ( ExprVal('max>={} if max is not None else None'.format(t), # title=str(t) + "℃", # name="max>=" + str(t), # units="days", # unit="day", # precision=0, # field=daily.MAX_TEMP, # description=str(t)+"℃ days"), # True ) for t in maxValuesToCheck ], #[ ( ExprVal('maxHumidex>max and maxHumidex>=' + str(t), # title=str(t) + " humidex", # name="humidex>=" + str(t), # units="days", # unit="day", # precision=0), # True ) for t in range(30, 51) ] #( FractionVal(daily.TOTAL_RAIN_MM, "Rain"), True ), (FractionVal(daily.TOTAL_SNOW_CM, "snow"), True), #( FractionVal(daily.TOTAL_PRECIP_MM, "precipitation"), True ), #( FractionVal(daily.AVG_WIND, "Wind"), False ), #( Avg(daily.MIN_TEMP, daily.MAX_TEMP, "temperature"), False ), #( FractionVal(daily.MEAN_HUMIDITY, "Humidity"), False ), ] monFilter = monthlyAverages.BeforeDateFilter(month=tomorrow.month, day=tomorrow.day) if allYear: monFilter = monthlyAverages.BeforeDateFilter(month=1, day=1) if allWinter: monFilter = WinterFilter() todayFilter = monthlyAverages.OneDayFilter(month=today.month, day=today.day) #import pudb; pu.db for field, cumulative in fieldList: thisyear = today.year if allWinter: thisyear = today.year if today.month >= 7 else today.year - 1 valByYear = monthlyAverages.yearDatas(monFilter, field=field, lastYear=thisyear, cumulative=cumulative) normalVal = monthlyAverages.normalMonthlyDatas(today.year, monFilter, field, cumulative=cumulative) if thisyear not in valByYear: continue ci80 = confidenceInterval80(valByYear, thisyear) ci50 = confidenceInterval50(valByYear, thisyear) thisYearVal = valByYear[thisyear] maxSince = None minSince = None for year in reversed(sorted(valByYear.keys())): if year != thisyear: val = valByYear[year] if maxSince is None and val >= thisYearVal: maxSince = year if minSince is None and val <= thisYearVal: minSince = year del val insideCi80 = (thisYearVal >= ci80.low and thisYearVal <= ci80.high) insideCi50 = (thisYearVal >= ci50.low and thisYearVal <= ci50.high) if field.units == 'days': # and False: todayValByYear = monthlyAverages.yearDatas(todayFilter, field=field, lastYear=today.year, cumulative=cumulative) if todayValByYear[today.year] == 0: #The countable event did not occur today, so skip. continue #import pdb; pdb.set_trace() yearByVal = reverseDict(valByYear) top5 = topNValuesLists(yearByVal, 5) if ((allYear or allWinter) and (inListOfLists(top5.values(), thisyear) or args.force)): tweetTopN(city, top5, field, thisyear) if justTop5: continue amountDescription = '' if thisYearVal < ci50[0] and (field.units != 'days' or thisYearVal != 0): amountDescription = 'just ' amount = formatWithUnits(thisYearVal, field, field.precision) if field.units == 'days': amount = formatWithUnits(thisYearVal, field, 0, skipUnits=True) if thisYearVal == 0: amount = "no" aboveBelow = 'above' if thisYearVal < normalVal.average: aboveBelow = 'below' if cumulative: deviation = deviationDescription(thisYearVal, normalVal.average, field) else: diff = abs(thisYearVal - normalVal.average) deviation = formatWithUnits(diff, field, field.precision) + ' ' + aboveBelow start = 'average' if cumulative: start = 'total' title = field.title.lower() ci80Low = formatWithUnits(ci80[0], field, field.precision, skipUnits=True) ci80High = formatWithUnits(ci80[1], field, field.precision) insideOutside = 'outside' if insideCi80: insideOutside = 'inside' cityName = stations.city[city].name avgWithUnits = formatWithUnits(normalVal.average, field, field.precision) #tweetText = ( # '#{cityName}\'s {start} {title} so far this year was' # ' {amountDescription}{amount},' # ' {deviation} {aboveBelow} average; {insideOutside} the normal range of' # ' {ci80Low} to {ci80High}'.format(**locals())) tweetText = ('#{cityName}\'s {start} {title} so far this year was' ' {amountDescription}{amount},' ' {deviation} the average of' ' {avgWithUnits}'.format(**locals())) if allYear: tweetText = ( '#{cityName}\'s {start} {title} during {today.year} was' ' {amountDescription}{amount},' ' {deviation} the average of' ' {avgWithUnits}'.format(**locals())) if field.units == 'days': units = field.unit if thisYearVal != 1: units = field.units nth = alert.nth(thisYearVal) #import pdb; pdb.set_trace() average = formatWithUnits(normalVal.average, field, precision=1) todayString = 'Today is' if today != daily.timeByCity[city].date(): todayString = 'Yesterday was' tweetText = ( '{todayString} #{cityName}\'s {nth} {title} day so far this year,' ' {deviation} the average of {average}.'.format(**locals())) #alertTweets.maybeTweetWithSuffix(city, tweetText) recordMin = normalVal.minimum recordMax = normalVal.maximum print( 'Record minimum was {recordMin}{field.units} in {normalVal.minYear}' .format(**locals())) print( 'Record maximum was {recordMax}{field.units} in {normalVal.maxYear}' .format(**locals())) plotIt = not insideCi80 or args.force if maxSince is None: print('***Max since records began in {}.'.format( min(valByYear.keys()))) plotIt = True elif today.year - maxSince > 10: maxSinceVal = valByYear[maxSince] print('***Max since {maxSince}:{maxSinceVal}{field.units}'.format( **locals())) plotIt = True if minSince is None: print('***Min since records began in {}.'.format( min(valByYear.keys()))) plotIt = True elif today.year - minSince > 10: minSinceVal = valByYear[minSince] print('***Min since {minSince}:{minSinceVal}{field.units}'.format( **locals())) plotIt = True if plotIt: fname = field.name pngname = "%s/Annual_%s.png" % (city, fname) alertTweets.maybeTweetWithSuffix(city, tweetText, fname=pngname) if todayMax is None: return todayAverageMax + 2 return max(todayMax, todayAverageMax + 2)
def main(city, dayToPlot, recently, thisDayInHistory, thisMonthInHistory, days, field, checkMax=True): hourField = hourlyFromDailyField[field] rows = [] plotData = [None] * 25 if field.minValue == 0: plotData[0] = 0 plotForecast = [None] * 25 today, _ = metarParse.synopticPeriod(datetime.datetime.utcnow()) if dayToPlot is not None: today = dayToPlot if type(dayToPlot) is str: today = datetime.date(*map(int, dayToPlot.split('-'))) snowPerDay = loadHourlyData(city, field, hourField, today) yesterday = today - datetime.timedelta(days=1) dailyData = daily.load(city) forecastData = forecast24Hour.getAndParse(city) todayUtcMidnight = datetime.datetime(today.year, today.month, today.day, hour=6, tzinfo=datetime.timezone.utc) mtz = stations.city[city].timezone todayUtcMidnightLocal = todayUtcMidnight.astimezone(mtz) if today in snowPerDay: if days == 2: plotData[0] = dailyData[yesterday].TOTAL_SNOW_CM maxVal, snowHours = snowPerDay[today] for utctime, vals in forecastData.items(): v = hourField(vals) if v is None: continue timediff = utctime - todayUtcMidnight index = timediff.days * 24 + timediff.seconds // 3600 if index < 0 or index > 24: continue rows.append(('{}h'.format(index), '{}{}'.format(v, field.units))) plotForecast[index] = {'value': v, 'node': {'r': 6}} if maxVal is None or v > maxVal: maxVal = v for hour, snow in sorted(snowHours.items()): totalSnow = snow if days == 2: totalSnow += dailyData[yesterday].TOTAL_SNOW_CM utcHour = hour.astimezone(datetime.timezone.utc) timediff = utcHour - todayUtcMidnight index = timediff.days * 24 + timediff.seconds // 3600 rows.append(('{}h'.format(hour), '{}{}'.format(snow, field.units))) plotData[index] = {'value': totalSnow, 'node': {'r': 6}} print( tabulate.tabulate(rows, headers=('Time', hourField.name), tablefmt="fancy_grid")) style = pygal.style.Style(label_font_size=15, major_label_font_size=20) line_chart = pygal.Line(style=style, print_values=True, x_label_rotation=80, include_x_axis=field.minValue == 0) cityName = stations.city[city].name line_chart.title = '{cityName} {field.name} on {today:%b %d, %Y}'.format( **locals()) line_chart.y_title = '{hourField.name} ({field.units})'.format( **locals()) line_chart.x_labels = map( lambda t: ('{:%H:%M}'.format(todayUtcMidnightLocal + datetime. timedelta(hours=t))), range(25)) #print(plotData) line_chart.add('Observed', plotData, stroke_style={'width': 3}, formatter=lambda t: str(t) + field.units) if plotForecast.count(None) < len(plotForecast): line_chart.add('Forecast', plotForecast, stroke_style={'width': 3}) historyLines = [] if recently: historyLines = snowiestDaysHistory(dailyData, today, field) if thisDayInHistory: historyLines = maxDaysToday(dailyData, today, field, maxValues=checkMax) if thisMonthInHistory: historyLines = maxDaysMonth(dailyData, today, field, maxValues=checkMax) labelByValue = {} for history in historyLines: fv = float(history.snow) labelByValue[fv] = labelByValue.get(fv, '') + history.label line_chart.add(history.label, ([{ 'value': history.snow, 'node': { 'r': 1 } }] + [None] * 23 + [history.snow]), stroke_style={'width': 3}, formatter=lambda t: '') pprint.PrettyPrinter().pprint(labelByValue) historyType = '' if thisDayInHistory: historyType = 'thisDayInHistory' if recently: historyType = 'recently' if thisMonthInHistory: historyType = 'thisMonthInHistory' minMax = 'max' if checkMax else 'min' fname = '{city}/{today}.hour-{field.name}.{minMax}.{historyType}.png'.format( **locals()) line_chart.render_to_png(fname, width=1024, height=768) return fname
#!/usr/bin/python3 # -*- coding: utf-8 -*- import daily import datetime field = daily.SNOW_ON_GRND_CM data = daily.load('ottawa') winterStartByYear = {} winterEndByYear = {} for year in range(data.minYear, data.maxYear + 1): daysCount = 0 missingDays = 0 days = daily.dayRange(datetime.date(year, 7, 1), datetime.date(year + 1, 7, 1)) for day in days: if day not in data or data[day].SNOW_ON_GRND_FLAG == 'M': if missingDays > 0: # Too many days missing, start from scratch daysCount = 0 missingDays = 0 else: missingDays += 1 daysCount += 1 continue if len(data[day].SNOW_ON_GRND_CM) > 0: depth = int(data[day].SNOW_ON_GRND_CM) if depth >= 1:
def createPlots(city, field, name, plotDate=None, plotZeros=True, verbose=True, output='svg'): units = field.units # findex = field.index # rawData = daily.load(city) rawForecast = forecast.getForecastDataEnvCan(city) # if verbose: print('annualData(..., field = %s, plotDate = %s)' % (field, plotDate)) # if plotDate == None: # if no date was provided, use the last date for which data is available plotDate = max(rawData.keys()) print("No plot-date was provided, using {}".format(plotDate)) # for (date, val) in rawForecast.items(): print(date, val) mergedData = daily.Data(rawData.copy()) for k, v in rawForecast.items(): if k not in mergedData: mergedData[k] = v plotData = annualData(rawData=mergedData, field=field, plotDate=plotDate) toDel = [] for year in plotData: value = plotData[year] if value == None: toDel.append(year) if plotZeros == False and value == 0: # We've been told to skip zeros, so we don't plot them toDel.append(year) for year in toDel: del plotData[year] del toDel #culledPlotData = makefit.makeFit(plotData, 75) plotLine, plotTop10, plotBottom10 = createPlot( plotData, city, name, '%s (%s)' % (field.englishName, units), field.englishName, units, plotDate, plotZeros, verbose) returnValue = plotLine, plotTop10, plotBottom10 yearByValue = {} valueByYear = {} recentValues = [] for year in plotData: # value = plotData[year] #if value == None: # value = 0 if (value not in yearByValue): yearByValue[value] = [year] else: yearByValue[value].append(year) valueByYear[year] = value if year < plotDate.year and year > plotDate.year - 31: recentValues.append(value) # if verbose: avg = sum(recentValues) / len(recentValues) print("Average is {:.1f}{}".format(float(avg), units)) # currentVariance = "unknown" if plotDate.year in valueByYear: thisYearValue = valueByYear[plotDate.year] if plotDate.year in valueByYear: currentVariance = "%.1f%s %s average" % ( abs(thisYearValue - avg), units, ["below", "above" ][thisYearValue > avg]) print("{} is {}".format(plotDate.year, currentVariance)) print([ float(v) for v in filter(lambda v: v < thisYearValue, recentValues) ]) print([ float(v) for v in filter(lambda v: v == thisYearValue, recentValues) ]) print([ float(v) for v in filter(lambda v: v > thisYearValue, recentValues) ]) # return returnValue
#!/usr/bin/python3 # -*- coding: utf-8 -*- import daily, sys, datetime from collections import namedtuple from namedList import * from fieldOperators import * now = datetime.datetime.now().date() weatherData = daily.load('ottawa') SumCount = namedList('SumCount', ('sum', 'count')) ValueWithYearA = namedStruct('ValueWithYear', 'value year') class ValueWithYear(ValueWithYearA): def __lt__(self, other): return self.value < other.value DistrobutionCount = namedStruct('DistrobutionCount', ('sum=0', 'count=0', 'average=None', 'min=None', 'max=None')) def JustJanuary(day): return day.month == 1 class JustAMonth: def __init__(self, month): self.month = month def __call__(self, day): return self.month == day.month def BeforeToday(day): return day.month < now.month or ( day.month == now.month and day.day < now.day)
#!/usr/bin/python # -*- coding: utf-8 -*- from __future__ import print_function import daily, time data = daily.load("farm") records = {} consequtiveRecords = 0 def reportRecords(dateTime): global consequtiveRecords if consequtiveRecords >= 3: print dateTime, consequtiveRecords consequtiveRecords = 0 for dateTime in sorted(data.keys()): if len(data[dateTime].MAX_TEMP) > 0: v = float(data[dateTime].MAX_TEMP) monthDay = (dateTime.month, dateTime.day) if monthDay not in records: records[monthDay] = v reportRecords(dateTime) elif records[monthDay] <= v: #print "Record %d %s %.1fC" % (dateTime.year, monthDay, v) records[monthDay] = v consequtiveRecords += 1 else: