def loadHourlyData(city, dayField, hourField, day):
if dayField == daily.TOTAL_SNOW_CM:
return metarParse.loadSnowWithHours(city)
elif dayField in (daily.MAX_TEMP, daily.MIN_TEMP):
return loadHourlyDataSwob(city, dayField, day)
mytimezone = stations.city[city].timezone
cityData = daily.load(city)
utcDayStart = datetime.datetime(day.year,
day.month,
day.day,
6,
tzinfo=datetime.timezone.utc)
dayStart = utcDayStart.astimezone(mytimezone)
dayEnd = dayStart + datetime.timedelta(days=1)
cityHourData = hourly.load(city, dateRange=(dayStart, dayEnd))
valByHour = {}
for utcHour, values in cityHourData.items():
v = hourField(values)
if v is None:
continue
l = utcHour.astimezone(mytimezone)
valByHour[l] = v
return {
day: metarParse.SnowAndHourly(cityData[day][dayField.index], valByHour)
}
def getValuesByYear(cityName, field, cumulative):
cityData = daily.load(cityName)
cityYearValues = annualAverages.getAnnualValues(cityData, field,
cumulative)
cityValuesByYear = {}
for year, value in cityYearValues:
cityValuesByYear[year] = value
return cityValuesByYear
def main(city, dayToPlot, recently, thisDayInHistory, field):
dailyData = daily.load(city)
dailyDataClean(dailyData)
if dayToPlot is None:
dayToPlot = datetime.date.today()
if recently:
historyLines = daysRecently(dailyData, dayToPlot, field)
else:
historyLines = daysToday(dailyData, dayToPlot, field)
if len(historyLines) == 1: #Most ever
startDate = dailyData.firstDateWithValue(field.index)
else:
startDate = historyLines[1].date
# - datetime.timedelta(days=(historyLines[0].date - historyLines[1].date).days * .1)
endDate = historyLines[0].date + datetime.timedelta(days=1)
plotData = []
for date in daily.dayRange(startDate,
endDate):
values = dailyData.get(date, None)
if values is None:
continue
plotData.append( (date, values[field.index]) )
style=pygal.style.Style(label_font_size=15, major_label_font_size=20)
date_chart = pygal.DateLine(style=style,
print_values=True,
#truncate_label=1000,
x_label_rotation=80)
date_chart.y_title = '{} ({})'.format(field.englishName,
field.units)
if field.minValue == 0:
date_chart.add(None, plotData, show_dots=False, fill=True)
else:
date_chart.add(None, plotData, show_dots=False)
labelByValue = {}
for history in historyLines[0:2]:
labelByValue[float(history.val)] = history.label
date_chart.add(None,
( ( startDate, history.val ),
( historyLines[0].date, history.val ) ),
formatter = lambda t: labelByValue[t[1]] )
historyType = ''
if thisDayInHistory:
historyType = 'thisDayInHistory'
if recently:
historyType = 'recently'
fname = '{city}/{dayToPlot}.recordSince.{historyType}.png'.format(**locals())
date_chart.render_to_png(fname,
width=1024, height=768)
return fname
def dispatch(cityName, firstYear,
endDate,
expression,
minRunLen,
showNth,
verbose=True,
skipIncomplete=False):
data = daily.load(cityName)
if firstYear != None:
for key in tuple(data.keys()):
if key.year < firstYear:
del data[key]
return count(cityName, data, expression, '??', endDate,
minRunLen,
showNth,
verbose, skipIncomplete=skipIncomplete)
def loadHourlyDataSwob(city, dayField, day):
if dayField == daily.TOTAL_SNOW_CM:
return metarParse.loadSnowWithHours(city)
mytimezone = stations.city[city].timezone
cityData = daily.load(city)
utcDayStart = datetime.datetime(day.year,
day.month,
day.day,
6,
tzinfo=datetime.timezone.utc)
dayStart = utcDayStart.astimezone(mytimezone)
dayEnd = dayStart + datetime.timedelta(days=1)
cityHourData = gatherSwob.parse(city)
hourly.load(city, dateRange=(dayStart, dayEnd))
valByHour = {}
for valuesDict in cityHourData:
ts = valuesDict['time']
if ts < dayStart or ts >= dayEnd:
continue
if ts == utcDayStart:
v = valuesDict['air_temp']
elif dayField == daily.MAX_TEMP:
v = valuesDict['max_air_temp_pst1hr']
elif dayField == daily.MIN_TEMP:
v = valuesDict['min_air_temp_pst1hr']
else:
assert (False)
if v is None:
continue
v = D(v)
l = ts.astimezone(mytimezone)
valByHour[l] = v
dayData = cityData.get(day, None)
if dayData is not None:
return {
day: metarParse.SnowAndHourly(dayData[dayField.index], valByHour)
}
return {day: metarParse.SnowAndHourly(None, valByHour)}
def sinceWhen(city, valueIndex, recordDate):
cityData = daily.load(city)
if recordDate is None:
firstDateWithObservation = cityData.firstDateWithValue(valueIndex)
sinceWhat = ( 'since records began in {begin}'
.format(begin=firstDateWithObservation.year) )
return sinceWhat
sinceWhat = 'since {}, {}'.format(dayDescription(city, recordDate), recordDate.year)
ageYears = yearsDiff(today(city), recordDate)
if ageYears < 1:
sinceWhat = 'since {}'.format(dayDescription(city, recordDate))
if ageYears > 2:
sinceWhat = ( "in more than {} years, since {}, {}"
.format(int(ageYears),
dayDescription(city, recordDate),
recordDate.year) )
if ageYears - int(ageYears) > 0.330:
sinceWhat = ( "in almost {} years, since {}, {}"
.format(int(ageYears+1),
dayDescription(city, recordDate),
recordDate.year) )
return sinceWhat
def main(city, year):
data = daily.load(city)
cold = ValWithDate(+999, None)
heat = ValWithDate(-999, None)
snow = ValWithDate(0, None)
rain = ValWithDate(0, None)
wind = ValWithDate(0, None)
for day, weather in data.items():
if day.year != year:
continue
#import pudb; pu.db
if (weather.MAX_TEMP is not None and weather.MAX_TEMP > heat.val):
heat = ValWithDate(weather.MAX_TEMP, day)
if (weather.MIN_TEMP is not None and weather.MIN_TEMP < cold.val):
cold = ValWithDate(weather.MIN_TEMP, day)
if (weather.TOTAL_SNOW_CM is not None
and weather.TOTAL_SNOW_CM > snow.val):
snow = ValWithDate(weather.TOTAL_SNOW_CM, day)
if (weather.TOTAL_RAIN_MM is not None
and weather.TOTAL_RAIN_MM > rain.val):
rain = ValWithDate(weather.TOTAL_RAIN_MM, day)
if (weather.SPD_OF_MAX_GUST_KPH is not None
and weather.SPD_OF_MAX_GUST_KPH > wind.val):
wind = ValWithDate(weather.SPD_OF_MAX_GUST_KPH, day)
cityName = stations.city[city].name
tweet = '#{cityName} 2016 extremes:'.format(**locals())
for name, u in (('heat', '℃'), ('cold', '℃'), ('snow', 'cm'),
('rain', 'mm'), ('wind', 'km/h')):
if locals()[name].date is None:
continue
tweet += ('\n{}: {}{} ({})'.format(
name,
locals()[name].val, u,
locals()[name].date.strftime('%b %d')))
alertTweets.maybeTweetWithSuffix(city, tweet)
#!/usr/bin/python3
import daily, time
import argparse
parser = argparse.ArgumentParser(
description='Calculate monthly average and charts.')
parser.add_argument('--city', default='ottawa')
parser.add_argument('--span')
args = parser.parse_args()
data = daily.load(args.city)
span = int(args.span)
days = []
maxSwing = 0
for date in reversed(sorted(data.keys())):
values = data[date]
if len(values.MAX_TEMP) > 0 and len(values.MIN_TEMP) > 0:
val = float(values.MIN_TEMP)
days.append(val)
val = float(values.MAX_TEMP)
days.append(val)
while len(days) > span * 2:
days.pop(0)
swing = max(days) - min(days)
if swing > maxSwing:
maxSwing = swing
print(date, swing, days)
def main(city, args, lastCheckedValue=None, today=None):
#import pdb; pdb.set_trace()
monthlyAverages.cityData = daily.load(city)
if today is None:
today = daily.timeByCity[city].date()
monthDate = today - datetime.timedelta(7)
nextMonthStart = datetime.date(
monthDate.year if monthDate.month < 12 else monthDate.year + 1,
monthDate.month + 1 if monthDate.month < 12 else 1, 1)
daysLeft = (nextMonthStart - today).days
monthlyAverages.now = monthDate
tomorrow = today + datetime.timedelta(days=1)
fieldList = []
if args.daysAbove:
fieldList += [(ExprVal('max>=' + str(t),
title=str(t) + "℃",
name="max>=" + str(t),
units="days",
unit="day",
precision=0,
field=daily.MAX_TEMP,
description=str(t) + "℃ days"), True)
for t in range(20, 29 + 1)]
if args.daysBelow:
fieldList += [(ExprVal('min<=' + str(t),
title=str(t) + "℃",
name="min<=" + str(t),
units="days",
unit="day",
precision=0,
field=daily.MIN_TEMP,
description=str(t) + "℃ nights"), True)
for t in range(-10, 1)]
#[ ( ExprVal('maxHumidex>max and maxHumidex>=' + str(t),
# title=str(t) + " humidex",
# name="humidex>=" + str(t),
# units="days",
# unit="day",
# precision=0),
# True ) for t in range(30, 51) ]
if args.rain:
if 'TOTAL_RAIN_MM' not in stations.city[args.city].skipDailyFields:
fieldList += [(FractionVal(daily.TOTAL_RAIN_MM, "Rain"), True)]
if args.snow:
fieldList += [(FractionVal(daily.TOTAL_SNOW_CM, "snow"), True)]
if args.snowDays:
fieldList += ([(ExprVal('snow>0',
title="snow",
name="snow",
units="days",
unit="day",
precision=0,
field=daily.TOTAL_SNOW_CM,
description="snow days"), True)] +
[(ExprVal('snow>=' + str(t),
title=str(t) + "cm snow",
name="snow>=" + str(t),
units="days",
unit="day",
precision=0,
field=daily.TOTAL_SNOW_CM,
description=str(t) + "cm snow days"), True)
for t in range(5, 41, 5)])
if args.wind:
fieldList += [(FractionVal(daily.AVG_WIND, "wind"), False)]
if args.meanTemp:
fieldList += [(ExprVal(
'meanTemp'
' if ("M" not in meanTempFlag and "I" not in meanTempFlag)'
' else ((max+min)/2'
' if ("M" not in minFlag and "I" not in minFlag'
' and "M" not in maxFlag and "I" not in maxFlag)'
' else None)',
name="Mean temperature",
title="Mean temperature",
units="℃",
unit="℃",
precision=2,
field=daily.MEAN_TEMP,
description='avg. hourly temp'), False)]
if args.avgTemp:
fieldList += [(Avg(daily.MIN_TEMP, daily.MAX_TEMP,
"Temperature"), False)]
if args.humidity:
fieldList += [(FractionVal(daily.MEAN_HUMIDITY, "Humidity"), False)]
monthStr = monthName(monthDate.month, long=True)
monFilter = monthlyAverages.MonthFilter(month=monthDate.month)
nextMonthYear = (monthDate.year * 12 + monthDate.month) // 12
nextMonthMonth = monthDate.month % 12 + 1
#import pudb; pu.db
for field, cumulative in fieldList:
valByYear = monthlyAverages.yearDatas(monFilter,
field=field,
lastYear=monthDate.year,
cumulative=cumulative)
normalVal = monthlyAverages.normalMonthlyDatas(monthDate.year,
monFilter,
field,
cumulative=cumulative)
if monthDate.year not in valByYear:
continue
ci80 = confidenceInterval80(valByYear, monthDate)
ci50 = confidenceInterval50(valByYear, monthDate)
thisYearVal = valByYear[monthDate.year]
maxSince = None
minSince = None
for year in reversed(sorted(valByYear.keys())):
if year != monthDate.year:
val = valByYear[year]
if maxSince is None and val >= thisYearVal:
maxSince = year
if minSince is None and val <= thisYearVal:
minSince = year
del val
maybeTweetTop5(city, valByYear, nextMonthYear, nextMonthMonth,
monthStr, field, monthDate)
maybeTweetMaxSince(city, valByYear, field, monthDate, monthStr,
maxSince, cumulative)
maybeTweetMinSince(city, valByYear, field, monthDate, monthStr,
minSince, cumulative)
insideCi80 = (thisYearVal >= ci80[0] and thisYearVal <= ci80[1])
insideCi50 = (thisYearVal >= ci50[0] and thisYearVal <= ci50[1])
amountDescription = ''
if thisYearVal < ci50[0] and (field.units != 'days'
or thisYearVal != 0):
amountDescription = 'just '
if daysLeft > 5 and thisYearVal > ci50[0]:
amountDescription = 'already '
amount = formatWithUnits(thisYearVal, field)
if field.units == 'days':
amount = formatWithUnits(thisYearVal, field, 0, skipUnits=True)
if thisYearVal == 0:
amount = "no"
aboveBelow = 'above'
if thisYearVal < normalVal.average:
aboveBelow = 'below'
if cumulative:
deviation = deviationDescription(thisYearVal, normalVal.average)
else:
diff = abs(float(thisYearVal - normalVal.average))
deviation = formatWithUnits(diff, field) + ' ' + aboveBelow
start = 'average'
if cumulative:
start = 'total'
title = field.title.lower()
ci80Low = formatWithUnits(ci80[0], field, skipUnits=True)
ci80High = formatWithUnits(ci80[1], field)
insideOutside = 'outside'
if insideCi80:
insideOutside = 'inside'
cityName = stations.city[city].name
average = formatWithUnits(normalVal.average,
field,
precision=max(field.precision, 1))
tweetText = ('#{cityName}\'s {start} {title} this {monthStr} was'
' {amountDescription}{amount},'
' {deviation} the average of {average}.'.format(
**locals()))
if daysLeft > 0:
tweetText = (
f'With {daysLeft} days left, #{cityName}\'s {start} {title} this {monthStr} is'
f' {amountDescription}{amount},'
f' {deviation} the average of {average}.')
if field.units == 'days':
units = field.unit
if thisYearVal != 1:
units = field.units
tweetText = (
'#{cityName} had'
' {amountDescription}{amount} {title} {units} this {monthStr},'
' {deviation} the average of {average}.'.format(**locals()))
print(tweetText)
#alertTweets.maybeTweetWithSuffix(city, tweetText)
recordMin = normalVal.minimum
recordMax = normalVal.maximum
print(
'Record minimum was {recordMin}{field.units} in {normalVal.minYear}'
.format(**locals()))
print(
'Record maximum was {recordMax}{field.units} in {normalVal.maxYear}'
.format(**locals()))
plotIt = not insideCi80 or args.force
if maxSince is None:
print('***Max since records began in {}.'.format(
min(valByYear.keys())))
plotIt = True
elif monthDate.year - maxSince > 10:
maxSinceVal = float(valByYear[maxSince])
print('***Max since {maxSince}:{maxSinceVal}{field.units}'.format(
**locals()))
plotIt = True
if minSince is None:
print('***Min since records began in {}.'.format(
min(valByYear.keys())))
plotIt = True
elif monthDate.year - minSince > 10:
minSinceVal = float(valByYear[minSince])
print('***Min since {minSince}:{minSinceVal}{field.units}'.format(
**locals()))
plotIt = True
if plotIt:
monthlyAverages.annualOrderedByMonth(city,
monFilter,
field,
cumulative=cumulative)
plotFname = snow.createPlot(
city,
running=cumulative,
field=field,
otheryears=(),
name=monthName(monthDate.month) +
title.replace(' ', '_').replace('℃', 'C'),
dataStartDay=datetime.date(monthDate.year, monthDate.month, 1),
plotStartDay=datetime.date(monthDate.year, monthDate.month, 1),
plotEndDay=datetime.date(nextMonthYear, nextMonthMonth, 1),
plotYMin=None)
pngname = plotFname.replace('/svg/', '/') + '.png'
command = 'rsvg-convert -o {pngname} --background-color=white {plotFname}'.format(
**locals())
print(command)
assert os.system(command) == 0
alertTweets.maybeTweetWithSuffix(city, tweetText, fname=pngname)
def main():
HISTORY = 10
parser = argparse.ArgumentParser(
description='Determine how often some weather occurs.')
parser.add_argument('expr', help='Which observation to check')
parser.add_argument('--city', default='ottawa')
parser.add_argument('--run', type=int, default=1)
parser.add_argument('-m', help='Mask', default=['*'], nargs='*')
parser.add_argument(
'--between',
help=
'Only consider dates between these two. Comma separated like 05-15,07-01'
)
parser.add_argument('--holiday', help='The name of a holiday')
parser.add_argument('--hour',
help='Use hourly data instead of daily.',
action='store_true',
default=False)
parser.add_argument('--winters',
help='Count by winter instead of by year.',
action='store_true',
default=False)
parser.add_argument('--group-by-year', action='store_true', default=False)
args = parser.parse_args()
run = args.run
city = args.city
fieldDict = {
'min': Value(daily.MIN_TEMP),
'max': Value(daily.MAX_TEMP),
'meanTemp': Value(daily.MEAN_TEMP),
'tempSpan': ValueDiff(daily.MAX_TEMP, daily.MIN_TEMP),
'rain': Value(daily.TOTAL_RAIN_MM),
'precip': Value(daily.TOTAL_PRECIP_MM),
'humidex': Value(daily.MAX_HUMIDEX),
'snow': Value(daily.TOTAL_SNOW_CM),
'snowpack': ValueEmptyZero(daily.SNOW_ON_GRND_CM),
'windgust': Value(daily.SPD_OF_MAX_GUST_KPH),
'wind': Value(daily.AVG_WIND),
'windchill': Value(daily.MIN_WINDCHILL),
'avgWindchill': Value(daily.MIN_WINDCHILL),
}
if args.hour:
fieldDict = {
'temp': Value(hourly.TEMP),
'dewpoint': Value(hourly.DEW_POINT_TEMP),
'humidity': Value(hourly.REL_HUM),
'windDir': Value(hourly.WIND_DIR),
'wind': Value(hourly.WIND_SPD),
'visibility': Value(hourly.VISIBILITY),
'pressure': Value(hourly.STN_PRESS),
'weather': ValueNoFlag(hourly.WEATHER),
'windchill': ValueNoFlag(hourly.WINDCHILL),
}
def allFieldNames():
return fieldDict.keys()
dateFilter = args.m
between = parseBetween(args.between)
if args.hour:
data = hourly.load(city)
for time, conditions in parseWeatherStatsRealtime.parse(city).items():
if time.minute == 0:
values = data.get(time, None)
if values is None:
data[time] = hourly.HourData(WEATHER=conditions)
else:
data[time] = values._replace(WEATHER=conditions)
for time, metarWeather in metarParse.genHourlyWeather(city):
if time.minute != 0:
continue
#if time.date() == dt.date(2017,3,27):
# import pudb; pu.db
values = data.get(time, None)
if values is None:
data[time] = hourly.HourData(WEATHER=metarWeather)
elif (values.WEATHER is None or values.WEATHER == 'NA'):
data[time] = values._replace(WEATHER=metarWeather)
else:
data = daily.load(city)
maxVal = None
curRun = deque()
curDates = deque()
fieldValues = fieldDict.values()
firstDate = None
matches = []
referencedValues = set()
compiledExpr = compile(args.expr, filename='.', mode='eval')
exprWords = pythonWords(args.expr)
for fieldName in fieldDict.keys():
if fieldName in exprWords:
referencedValues.add(fieldName)
#print(referencedValues)
#print(tuple(allFieldNames()))
class refCountedList(deque):
def __init__(self):
self.indexSet = set()
def __getitem__(self, ind):
if type(ind) != slice: #print('[{ind}]'.format(**locals()))
self.indexSet.add(ind)
return deque.__getitem__(self, ind)
return list(self)[ind]
def clearIndexSet(self):
self.indexSet.clear()
historyDates = deque([])
history = refCountedList()
class History():
pass
expectedDiff = dt.timedelta(days=1)
if args.hour:
expectedDiff = dt.timedelta(hours=1)
mytimezone = stations.city[args.city].timezone
for date in sorted(data.keys()):
if args.hour:
utchour = date
localhour = utchour.astimezone(mytimezone)
date = localhour
if (len(historyDates) > 0 and date - historyDates[0] != expectedDiff):
historyDates.clear()
history.clear()
if matchDate(date, args.m, between, args.holiday):
#if date.year == 2016 and date.month == 11 and date.day == 30:
# import pudb; pu.db
vals = {
'history': history,
"__builtins__": __builtins__,
'time': date
}
history.appendleft(History())
historyDates.appendleft(date)
for fieldName, fieldCall in fieldDict.items():
vals[fieldName] = fieldCall(data[date], date)
flagValue = fieldCall.getFlag(data[date])
if flagValue is not None:
vals[fieldName + 'Flag'] = flagValue
if type(vals[fieldName]) is not SpecialNone:
history[0].__setattr__(fieldName, vals[fieldName])
if flagValue is not None:
history[0].__setattr__(fieldName + 'Flag', flagValue)
skip = False
usedVals = {}
for fieldName in referencedValues:
if type(vals[fieldName]) is SpecialNone:
#print('Skipping {} because {} is None'.format(date, fieldName))
skip = True
break
#print(fieldName, type(vals[fieldName]) is SpecialNone)
usedVals[fieldName] = vals[fieldName]
if skip:
continue
if firstDate is None:
firstDate = date
lastDate = date
#expr = args.expr.format(**vals)
#print(date, args.expr, usedVals, vals)
history.clearIndexSet()
try:
val = eval(compiledExpr, vals)
except IndexError:
val = False
except AttributeError:
val = False
except TypeError:
for offset, date in enumerate(historyDates):
print(date)
for name in allFieldNames():
if hasattr(history[offset], name):
print(name, history[offset].__getattribute__(name))
raise
#print(val)
if val is True or type(val) is tuple and val[0] is True:
#print('+++')
#for offset, date in enumerate(historyDates):
# print(date)
# for name in allFieldNames():
# if name in dir(history[offset]):
# print(name, history[offset].__getattribute__(name))
#print('---')
for i in history.indexSet:
for fieldName in referencedValues:
usedVals['history[{}].{}'.format(i, fieldName)] = (
history[i].__getattribute__(fieldName))
#print(history.indexSet)
expectedDate = date
if len(curDates) > 0:
expectedDate = curDates[-1] + expectedDiff
if date != expectedDate:
#print('Clearing run', date, curDates[-1])
curDates = deque([date])
curRun = deque([usedVals])
if type(val) is tuple:
curRun = deque([{'expr': val[1]}])
else:
curDates.append(date)
if type(val) is tuple:
curRun.append({'expr': val[1]})
else:
curRun.append(usedVals)
if len(curRun) > run:
curRun.popleft()
curDates.popleft()
#print(date, len(curRun), run)
if len(curRun) == run:
printDate(curDates)
if len(curRun[0]) == 1:
print(' '.join(
[str(first(a.values())) for a in curRun]))
else:
print(curRun)
matches.append(curDates)
if args.winters:
grouping = tuple([winterFromDate(a[0]) for a in matches])
group = 'winter'
else:
grouping = tuple([a[0].year for a in matches])
group = 'year'
if args.group_by_year:
print(tuple(enumerate(Counter(grouping).most_common(30))))
print('Total count: {}'.format(len(grouping)))
print('Occurance: ', end='')
yearSpan = lastDate.year - firstDate.year + 1
if len(matches) < yearSpan:
print('once every {:.1f} {}s'.format(yearSpan / len(matches), group))
else:
print('{:.1f} times per {}'.format(len(matches) / yearSpan, group))
print('Yearly Occurance:', end=' ')
uniqGrouping = sorted(list(set(grouping)))
if len(uniqGrouping) < yearSpan:
occurance = yearSpan / len(uniqGrouping)
print('once every {occurance:.1f} {group}s,'.format(**locals()),
end=' ')
else:
occurance = len(uniqGrouping) / yearSpan
print('{occurance:.1f} times per {group},'.format(**locals()), end=' ')
recentFilter = lambda t: t >= today().year - 30 and t < today().year
if args.winters:
recentFilter = lambda t: t >= thisWinter() - 30 and t < thisWinter()
recentOccursYears = tuple(filter(recentFilter, grouping))
recentUniqYears = tuple(sorted(set(recentOccursYears)))
print('{} out of the past 30 years: {}'.format(len(recentUniqYears),
str(recentUniqYears)))
recentYears = range(thisYear() - 30, thisYear())
if args.winters:
recentYears = range(thisWinter() - 30, thisWinter())
recentLen = len(recentUniqYears)
recentYearCounter = Counter(recentOccursYears)
recentLens = sorted(recentYearCounter[a] for a in recentYears)
print('{} during the past 30 years'.format(recentLen))
if True: #recentLen > 30:
print('Average is {:.1f}/year during the past 30 years'.format(
len(recentOccursYears) / 30))
print('Median is {},{}/year during the past 30 years'.format(
recentLens[int(len(recentLens) / 2)], recentLens[int(
(len(recentLens) + 1) / 2)]))
print(
'80% CI is {},{}'.format(
recentLens[len(recentLens) // 10],
recentLens[len(recentLens) - (len(recentLens) // 10) - 1]),
recentLens)
#print(sorted([(len(o),y) for y,o in recentByYear.items()]))
else:
print('Every {:.1f} years during the past 30 years'.format(30 /
recentLen))
occurDays = '∞'
if recentLen > 0:
occurDays = (dt.date.today() - yearsAgo(30)).days / recentLen
print('Every {} days during the past 30 years'.format(occurDays))
if recentLen > 0:
print('Every {} months, {:.1f} days during the past 30 years'.format(
int(occurDays // 30), occurDays % 30))
if args.hour:
recentOccur = tuple(
filter(lambda t: t[0].date() >= yearsAgo(1), matches))
else:
recentOccur = tuple(filter(lambda t: t[0] >= yearsAgo(1), matches))
print('{} during the past year'.format(len(recentOccur)))
if args.winters:
thisWinterCount = grouping.count(thisWinter())
print('{} so far this winter'.format(thisWinterCount))
else:
thisYearCount = grouping.count(thisYear())
print('{} so far this year'.format(thisYearCount))
'humidex': Value(daily.MAX_HUMIDEX),
'snow': Value(daily.TOTAL_SNOW_CM),
'snowpack': ValueEmptyZero(daily.SNOW_ON_GRND_CM),
'windgust': Value(daily.SPD_OF_MAX_GUST_KPH),
'wind': Value(daily.AVG_WIND),
'windchill': Value(daily.MIN_WINDCHILL),
'avgWindchill': Value(daily.AVG_WINDCHILL),
'meanHumidity': Value(daily.MEAN_HUMIDITY),
'minHumidity': Value(daily.MIN_HUMIDITY),
'minRec': ValueNearRecordMin(daily.MIN_TEMP),
}[fieldName]
dateFilter = args.m
between = parseBetween(args.between)
data = daily.load(city)
maxVal = -999
curRun = collections.deque([-999] * run, run)
curDates = collections.deque([datetime.date(1800, 1, 1)] * run, run)
rawData = open('recordSince.csv', 'w')
for date in reversed(sorted(data.keys())):
if matchDate(date, [dateFilter], between, args.holiday):
flag = field.getFlag(data[date])
if 'H' in flag and field.getName() in ('MIN_TEMP', 'MIN_WINDCHILL'):
continue
val = field(data[date], date)
if val != None:
ret = MaxMin(dayInfo.max, dayInfo.min)
maxMinCache[key] = ret
return ret
def getMonthRecords(city, targetYear, field):
ret = MaxMin(max=[], min=[])
for i in range(12):
ret.max.append(Record(-999, year=1900))
ret.min.append(Record(+999, year=1900))
cityData = daily.dataByCity[city]
for date in sorted(cityData.keys()):
if date.year == targetYear:
break
val = cityData[date][field.index]
if val is not None:
if val > ret.max[date.month - 1].value:
ret.max[date.month - 1] = Record(val, date.year, date.month,
date.day)
if val < ret.min[date.month - 1].value:
ret.min[date.month - 1] = Record(val, date.year, date.month,
date.day)
return ret
if __name__ == '__main__':
daily.load("hamilton")
print(getInfo('hamilton', datetime.date(2017, 3, 1), daily.AVG_WIND))
from monthName import monthName
now = datetime.date.today() - datetime.timedelta(3)
cityName = 'ottawa'
firstYear = None
opts, args = getopt.getopt(sys.argv[1:], 'c:f:', ['city=', 'firstYear='])
for opt, arg in opts:
if opt in ('-c','--city'):
cityName = arg
elif opt in ('-f','--firstYear'):
firstYear = int(arg)
cityData = daily.load(cityName)
if firstYear != None:
keys = cityData.keys()
for key in keys:
if key.year < firstYear:
del cityData[key]
class YearMonthFilter():
__slots__ = ()
def __init__(self, year, month):
self.year = year
self.month = month
#
def __call__(self, year = None):
if year != None:
def first(cityName,
expr,
name,
dateRange,
excludeThisYear=False,
order="first",
yaxisLabel=None,
run=1,
limitToMostRecentYears=None,
verboseIfInDateRange=None,
):
data = daily.load(cityName)
if limitToMostRecentYears != None:
ndata = daily.Data()
yearsAgo = datetime.date(now.year-limitToMostRecentYears, now.month, now.day)
for key in data:
if key >= yearsAgo:
ndata[key] = data[key]
data = ndata
fieldDict = {
'min': Value(daily.MIN_TEMP),
'max': Value(daily.MAX_TEMP),
'tempSpan': ValueDiff(daily.MAX_TEMP, daily.MIN_TEMP),
'rain': Value(daily.TOTAL_RAIN_MM),
'humidex': Value(daily.MAX_HUMIDEX),
'snow': Value(daily.TOTAL_SNOW_CM),
'snowpack': ValueEmptyZero(daily.SNOW_ON_GRND_CM),
'windgust': Value(daily.SPD_OF_MAX_GUST_KPH),
'wind': Value(daily.AVG_WIND),
'windchill': Value(daily.MIN_WINDCHILL),
'avgWindchill': Value(daily.MIN_WINDCHILL),
}
fieldValues = fieldDict.values()
referencedValues=[]
for fieldName in fieldDict.keys():
if '{'+fieldName+'}' in expr:
referencedValues.append(fieldName)
firstByYear = {}
eThisYear = now.year
endTimeThisYear = datetime.date(now.year, dateRange.endMonth, dateRange.endDay)
if dateRange.yearCross() and now < endTimeThisYear:
eThisYear -= 1
for baseyear in range(data.minYear, data.maxYear-dateRange.todayIsYearCross()+1):
try:
dayOffset = datacache.readCache(cityName, baseyear,
'{}.{}.{}.{}'.format(name,order,str(dateRange),expr))
if dayOffset != None:
firstByYear[baseyear] = dayOffset
continue
except datacache.NotInCache:
pass
starttime = datetime.date(baseyear, dateRange.startMonth, dateRange.startDay)
endtime = datetime.date(baseyear+dateRange.yearCross(), dateRange.endMonth, dateRange.endDay)
#print baseyear, starttime, endtime
dayRange = daily.dayRange(starttime,endtime)
if order=="last":
dayRange = daily.dayRange(endtime-datetime.timedelta(1), starttime-datetime.timedelta(1), -1)
if excludeThisYear and baseyear == now.year-dateRange.yearCross():
# don't consider this year when looking for records for last event, because this year is not over
continue
expectedDayCount = 0
observedDayCount = 0
for day in dayRange:
expectedDayCount += 1
if day in data:
observedDayCount += 1
else:
continue
if baseyear in firstByYear:
# We already figured out which day was first
continue
vals = {}
for fieldName, fieldCall in fieldDict.items():
try:
vals[fieldName] = fieldCall(data[day], day)
vals[fieldName+'Flag'] = '"' + fieldCall.getFlag(data[day]) + '"'
except KeyError:
#print day, 'KeyError'
vals[fieldName] = None
skip=False
usedVals={}
for fieldName in fieldDict.keys():
if fieldName in referencedValues:
if vals[fieldName] is None:
#print 'Skipping {} because {} is None'.format(day, fieldName)
skip=True
break
usedVals[fieldName] = vals[fieldName]
if skip:
continue
#resolvedExpr = expr.format(**vals)
#print(vals)
#val = eval(expr)
val = eval(expr, vals)
#if True: #day == datetime.date(2015,10,17):
#print day, resolvedExpr, val
if val is True:
dayOffset = (day - starttime).days
firstByYear[baseyear] = dayOffset
break
observedPercent = observedDayCount * 100 / expectedDayCount
if observedPercent < 85 and baseyear != eThisYear:
print('Skipping {baseyear} because it only had {observedPercent:.1f}% of the days'.format(**locals()))
if baseyear in firstByYear:
firstByYear.pop(baseyear)
elif baseyear not in firstByYear and baseyear != eThisYear:
print('Event did not happen during {}.'.format(baseyear))
datacache.cacheThis(cityName, baseyear,
'{}.{}.{}.{}'.format(name,order,str(dateRange),expr),
firstByYear.get(baseyear, None))
yearByFirst = reverseDict(firstByYear)
#for offset in sorted(yearByFirst.keys()):
# for year in yearByFirst[offset]:
# print datetime.date(year, dateRange.startMonth, dateRange.startDay) + datetime.timedelta(offset)
#print yearByFirst
verbose = False
if verboseIfInDateRange == None:
verbose = True
elif eThisYear in firstByYear:
thisYearFirstDayOffset = firstByYear[eThisYear]
firstThisYearDate = (
datetime.date(eThisYear, dateRange.startMonth, dateRange.startDay)
+ datetime.timedelta(thisYearFirstDayOffset) )
if ( firstThisYearDate >= verboseIfInDateRange[0]
and firstThisYearDate <= verboseIfInDateRange[1] ):
verbose = True
(earliest, secondEarliest, *ignore) = sorted(filter(lambda y: y!=now.year-dateRange.yearCross(), yearByFirst.keys()))[:2] + [None,None]
(secondLatest, latest, *ignore) = sorted(filter(lambda y: y!=now.year-dateRange.yearCross(), yearByFirst.keys()))[-2:] + [None,None]
earliestYears = lookupEmptyListIfNone(yearByFirst, earliest)
secondEarliestYears = lookupEmptyListIfNone(yearByFirst, secondEarliest)
latestYears = lookupEmptyListIfNone(yearByFirst, latest)
secondLatestYears = lookupEmptyListIfNone(yearByFirst, secondLatest)
earliestDates=[]
latestDates=[]
if verbose:
earliestDates=showRecords(name, 'earliest', earliestYears, firstByYear, dateRange)
showRecords(name, '2nd earliest', secondEarliestYears, firstByYear, dateRange)
showRecords(name, '2nd latest', secondLatestYears, firstByYear, dateRange)
latestDates=showRecords(name, 'latest', latestYears, firstByYear, dateRange)
avgKeys = filter(lambda t: t in firstByYear, range(eThisYear-30, eThisYear))
offsets = sorted([firstByYear[a] for a in avgKeys])
avg = calcAvg(offsets)
median = calcMedian(offsets)
avgFirst = calcDate(eThisYear, dateRange.startMonth, dateRange.startDay, avg)
medianFirst = calcDate(eThisYear, dateRange.startMonth, dateRange.startDay, median)
if verbose:
print("average {name} is {avgFirst}".format(**locals()))
print("median {name} is {medianFirst}".format(**locals()))
ret = None
if eThisYear in firstByYear:
thisYearFirst = firstByYear[eThisYear]
countEarlier = 0
countEqual = 0
countLater = 0
recentCountEarlier = []
recentCountEqual = []
recentCountLater = []
for first in sorted(yearByFirst.keys()):
if first < thisYearFirst:
countEarlier += len(yearByFirst[first])
recentCountEarlier += filter(lambda y: y<eThisYear and y > eThisYear-31, yearByFirst[first])
elif first == thisYearFirst:
countEqual += len(yearByFirst[first]) - 1 #Subtract the current year
recentCountEqual += filter(lambda y: y<eThisYear and y > eThisYear-31, yearByFirst[first])
else:
countLater += len(yearByFirst[first])
recentCountLater += filter(lambda y: y<eThisYear and y > eThisYear-31, yearByFirst[first])
totalCount = countEarlier + countEqual + countLater
totalRecentCount = len(recentCountEarlier) + len(recentCountEqual) + len(recentCountLater)
firstThisYear = ( datetime.date(eThisYear, dateRange.startMonth, dateRange.startDay)
+ datetime.timedelta(thisYearFirst) )
if verbose:
print('%s %s was %s' % (now.year, name, firstThisYear ))
print('%d%% of last %d years were earlier.'
% (round(countEarlier * 100.0 / totalCount), totalCount))
print('%d%% of last %d years were the same.'
% (round(countEqual * 100.0 / totalCount), totalCount))
print('%d%% of last %d years were later.'
% (round(countLater * 100.0 / totalCount), totalCount))
print('%d of last %d years were earlier.'
% (len(recentCountEarlier), totalRecentCount), sorted(recentCountEarlier))
print('%d of last %d years were the same.'
% (len(recentCountEqual), totalRecentCount), sorted(recentCountEqual))
print('%d of last %d years were later.'
% (len(recentCountLater), totalRecentCount), sorted(recentCountLater))
ret = firstThisYear, medianFirst, earliestDates, latestDates
else:
print('Not showing this year because eThisYear="{}" and firstByYear="{}"'.format(eThisYear, firstByYear))
if len(yearByFirst) == 0:
# There's nothing to plot, so don't even bother trying
return ret
plotDataOtherYears = []
plotDataThisYear = []
dateLabels = []
for key in sorted(yearByFirst.keys()):
if eThisYear in yearByFirst[key]:
plotDataThisYear.append((key, len(yearByFirst[key]), '#%s' % ','.join(map(str, yearByFirst[key])) ))
else:
plotDataOtherYears.append((key, len(yearByFirst[key]), '#%s' % ','.join(map(str, yearByFirst[key]))))
histogramFname = '%s/svg/%s' % (cityName, name.replace(' ', '_'))
if ret != None:
ret = ret + (histogramFname,)
plot = gnuplot.Plot(histogramFname)
dateMin = min(yearByFirst.keys())
dateMax = max(yearByFirst.keys())
plotDateMin = dateMin-1 #int(dateMin - (dateMax - dateMin)*.25)
plotDateMax = dateMax+1 #int(dateMax + (dateMax - dateMin)*.25)
for dayOffset in range(plotDateMin, plotDateMax+1, 1):
date = datetime.date(now.year, dateRange.startMonth, dateRange.startDay) + datetime.timedelta(dayOffset)
if date.day % 5 == 1 and date.day != 31:
dateLabels.append('"%s" %d' % (date.strftime('%b/%d'), dayOffset))
ylabel = 'Number of years when %s happened on this day' % name
if yaxisLabel != None:
ylabel = yaxisLabel
plot.open(title='%s in %s' % (name, cityName.capitalize()),
ylabel=ylabel,
xmin=plotDateMin,
xmax=plotDateMax,
ymin=0,
ymax=max(len(a) for a in yearByFirst.values())+1,
xtics=dateLabels, xticsRotate=45,
margins=[7,8,2,5])
plot.addLine(gnuplot.Line('Other years', plotDataOtherYears, plot='boxes'))
plot.addLine(gnuplot.Line('This year', plotDataThisYear, plot='boxes'))
plot.plot()
plot.close()
print('')
plotdata = []
for (year, plotCount) in firstByYear.items():
if plotCount != None:
plotdata.append(
(year, plotCount,
( '#'
+str(datetime.date(year, dateRange.startMonth, dateRange.startDay)
+ datetime.timedelta(plotCount)))))
ytics = []
for m in range(1,13):
for d in (1,10,20):
thisDate = datetime.date(2015,m,d)
dayOffset = (thisDate - datetime.date(2015, dateRange.startMonth, dateRange.startDay)).days
ytics.append('"%s" %d' % (thisDate.strftime('%b %d'), dayOffset))
#print(tuple(t[:2] for t in plotdata))
lineFit = linear.linearTrend(tuple(t[:2] for t in plotdata))
plot = gnuplot.Plot("%s/svg/%s_%s-%u_%s.line"
% (cityName, name.replace(' ', '_'),
monthName(dateRange.startMonth), dateRange.startDay,
dateRange.lastDay().replace(' ', '-') ),
yaxis=2)
plot.open(title='%s %s between %s %u and %s'
% (cityName.capitalize(), name,
monthName(dateRange.startMonth), dateRange.startDay,
dateRange.lastDay() ),
ytics=ytics)
plot.addLine(gnuplot.Line("Linear", lineFit, lineColour='green', plot='lines'))
plot.addLine(gnuplot.Line("Date", plotdata, lineColour='purple'))
plot.plot()
plot.close()
return ret
import daily, datetime
def meanstdv(x):
from math import sqrt
n, mean, std = len(x), 0, 0
for a in x:
mean = mean + a
mean = mean / float(n)
for a in x:
std = std + (a - mean)**2
std = sqrt(std / float(n - 1))
return mean, std
weatherData = daily.load('farm')
rawData = {}
for year in range(weatherData.minYear, weatherData.maxYear + 1):
rawData[year] = []
startDay = datetime.date(year, 1, 1)
endDay = datetime.date(year, 2, 1)
for date in daily.dayRange(startDay, endDay):
try:
dayValues = weatherData[date]
except KeyError:
continue
v = dayValues.MAX_TEMP
#!/usr/bin/python
from __future__ import print_function
import sys, datetime, daily, ps, forecast, sys, dailyRecords
from collections import namedtuple
city = sys.argv[1]
historyDays = int(sys.argv[2])
now = forecast.now
daily.load(city)
Temp_MinMax = namedtuple('Temp_MinMax', 'min max')
def getTempData():
history = Temp_MinMax({}, {})
for daysOffset in range(-historyDays, 8 + 1):
date = now + datetime.timedelta(daysOffset)
history.min[date] = dailyRecords.getInfo(city, date, daily.MIN_TEMP)
history.max[date] = dailyRecords.getInfo(city, date, daily.MAX_TEMP)
return history
class COLDER():
__slots__ = ()
class WARMER():
#!/usr/bin/python
# -*- coding: utf-8 -*-
from __future__ import print_function
import time, posix, daily
data = daily.load("ottawa")
class FloatValue():
__slots__ = ()
def __init__(self, field):
self.fieldIndex = field.index
def __call__(self, fields):
r = fields[self.fieldIndex]
if len(r) == 0:
return None
return float(r)
class IntValue():
__slots__ = ()
def __init__(self, field):
self.fieldIndex = field.index
def __call__(self, fields):
r = fields[self.fieldIndex]
if len(r) == 0:
return None
def main(
city,
force=False,
lastCheckedValue=None,
today=None,
maxValueToCheck=None,
allYear=False,
allWinter=False,
justTop5=False,
dataSinceDay=None,
):
#import pdb; pdb.set_trace()
data = daily.load(city)
if dataSinceDay is not None:
for day in tuple(data.keys()):
if day < dataSinceDay:
del data[day]
monthlyAverages.cityData = data
if today is None:
today = daily.timeByCity[city].date()
#if allYear:
# today = dt.date(daily.timeByCity[city].date().year, 12, 31)
yearToCheck = (today - dt.timedelta(7)).year
if yearToCheck != today.year:
today = dt.date(yearToCheck, 12, 31)
monthlyAverages.now = today
tomorrow = today + dt.timedelta(days=1)
todayMaxInfo = dailyRecords.getInfo(city, today, daily.MAX_TEMP)
todayAverageMax = roundAwayFromZero(todayMaxInfo.normal)
todayMax = None
if todayMaxInfo.recent is not None:
todayMax = int(todayMaxInfo.recent)
if lastCheckedValue is None:
minValueToCheck = int(todayAverageMax) + 2
else:
minValueToCheck = int(lastCheckedValue) + 1
if maxValueToCheck is None:
maxValueToCheck = todayMax
if todayMax is None:
maxValueToCheck = 35
maxValuesToCheck = filter(lambda t: t % 10 == 0,
range(minValueToCheck, maxValueToCheck + 1))
fieldList = [
#*[ ( ExprVal('max>={} if max is not None else None'.format(t),
# title=str(t) + "℃",
# name="max>=" + str(t),
# units="days",
# unit="day",
# precision=0,
# field=daily.MAX_TEMP,
# description=str(t)+"℃ days"),
# True ) for t in maxValuesToCheck ],
#[ ( ExprVal('maxHumidex>max and maxHumidex>=' + str(t),
# title=str(t) + " humidex",
# name="humidex>=" + str(t),
# units="days",
# unit="day",
# precision=0),
# True ) for t in range(30, 51) ]
#( FractionVal(daily.TOTAL_RAIN_MM, "Rain"), True ),
(FractionVal(daily.TOTAL_SNOW_CM, "snow"), True),
#( FractionVal(daily.TOTAL_PRECIP_MM, "precipitation"), True ),
#( FractionVal(daily.AVG_WIND, "Wind"), False ),
#( Avg(daily.MIN_TEMP, daily.MAX_TEMP, "temperature"), False ),
#( FractionVal(daily.MEAN_HUMIDITY, "Humidity"), False ),
]
monFilter = monthlyAverages.BeforeDateFilter(month=tomorrow.month,
day=tomorrow.day)
if allYear:
monFilter = monthlyAverages.BeforeDateFilter(month=1, day=1)
if allWinter:
monFilter = WinterFilter()
todayFilter = monthlyAverages.OneDayFilter(month=today.month,
day=today.day)
#import pudb; pu.db
for field, cumulative in fieldList:
thisyear = today.year
if allWinter:
thisyear = today.year if today.month >= 7 else today.year - 1
valByYear = monthlyAverages.yearDatas(monFilter,
field=field,
lastYear=thisyear,
cumulative=cumulative)
normalVal = monthlyAverages.normalMonthlyDatas(today.year,
monFilter,
field,
cumulative=cumulative)
if thisyear not in valByYear:
continue
ci80 = confidenceInterval80(valByYear, thisyear)
ci50 = confidenceInterval50(valByYear, thisyear)
thisYearVal = valByYear[thisyear]
maxSince = None
minSince = None
for year in reversed(sorted(valByYear.keys())):
if year != thisyear:
val = valByYear[year]
if maxSince is None and val >= thisYearVal:
maxSince = year
if minSince is None and val <= thisYearVal:
minSince = year
del val
insideCi80 = (thisYearVal >= ci80.low and thisYearVal <= ci80.high)
insideCi50 = (thisYearVal >= ci50.low and thisYearVal <= ci50.high)
if field.units == 'days': # and False:
todayValByYear = monthlyAverages.yearDatas(todayFilter,
field=field,
lastYear=today.year,
cumulative=cumulative)
if todayValByYear[today.year] == 0:
#The countable event did not occur today, so skip.
continue
#import pdb; pdb.set_trace()
yearByVal = reverseDict(valByYear)
top5 = topNValuesLists(yearByVal, 5)
if ((allYear or allWinter)
and (inListOfLists(top5.values(), thisyear) or args.force)):
tweetTopN(city, top5, field, thisyear)
if justTop5:
continue
amountDescription = ''
if thisYearVal < ci50[0] and (field.units != 'days'
or thisYearVal != 0):
amountDescription = 'just '
amount = formatWithUnits(thisYearVal, field, field.precision)
if field.units == 'days':
amount = formatWithUnits(thisYearVal, field, 0, skipUnits=True)
if thisYearVal == 0:
amount = "no"
aboveBelow = 'above'
if thisYearVal < normalVal.average:
aboveBelow = 'below'
if cumulative:
deviation = deviationDescription(thisYearVal, normalVal.average,
field)
else:
diff = abs(thisYearVal - normalVal.average)
deviation = formatWithUnits(diff, field,
field.precision) + ' ' + aboveBelow
start = 'average'
if cumulative:
start = 'total'
title = field.title.lower()
ci80Low = formatWithUnits(ci80[0],
field,
field.precision,
skipUnits=True)
ci80High = formatWithUnits(ci80[1], field, field.precision)
insideOutside = 'outside'
if insideCi80:
insideOutside = 'inside'
cityName = stations.city[city].name
avgWithUnits = formatWithUnits(normalVal.average, field,
field.precision)
#tweetText = (
# '#{cityName}\'s {start} {title} so far this year was'
# ' {amountDescription}{amount},'
# ' {deviation} {aboveBelow} average; {insideOutside} the normal range of'
# ' {ci80Low} to {ci80High}'.format(**locals()))
tweetText = ('#{cityName}\'s {start} {title} so far this year was'
' {amountDescription}{amount},'
' {deviation} the average of'
' {avgWithUnits}'.format(**locals()))
if allYear:
tweetText = (
'#{cityName}\'s {start} {title} during {today.year} was'
' {amountDescription}{amount},'
' {deviation} the average of'
' {avgWithUnits}'.format(**locals()))
if field.units == 'days':
units = field.unit
if thisYearVal != 1:
units = field.units
nth = alert.nth(thisYearVal)
#import pdb; pdb.set_trace()
average = formatWithUnits(normalVal.average, field, precision=1)
todayString = 'Today is'
if today != daily.timeByCity[city].date():
todayString = 'Yesterday was'
tweetText = (
'{todayString} #{cityName}\'s {nth} {title} day so far this year,'
' {deviation} the average of {average}.'.format(**locals()))
#alertTweets.maybeTweetWithSuffix(city, tweetText)
recordMin = normalVal.minimum
recordMax = normalVal.maximum
print(
'Record minimum was {recordMin}{field.units} in {normalVal.minYear}'
.format(**locals()))
print(
'Record maximum was {recordMax}{field.units} in {normalVal.maxYear}'
.format(**locals()))
plotIt = not insideCi80 or args.force
if maxSince is None:
print('***Max since records began in {}.'.format(
min(valByYear.keys())))
plotIt = True
elif today.year - maxSince > 10:
maxSinceVal = valByYear[maxSince]
print('***Max since {maxSince}:{maxSinceVal}{field.units}'.format(
**locals()))
plotIt = True
if minSince is None:
print('***Min since records began in {}.'.format(
min(valByYear.keys())))
plotIt = True
elif today.year - minSince > 10:
minSinceVal = valByYear[minSince]
print('***Min since {minSince}:{minSinceVal}{field.units}'.format(
**locals()))
plotIt = True
if plotIt:
fname = field.name
pngname = "%s/Annual_%s.png" % (city, fname)
alertTweets.maybeTweetWithSuffix(city, tweetText, fname=pngname)
if todayMax is None:
return todayAverageMax + 2
return max(todayMax, todayAverageMax + 2)
def main(city,
dayToPlot,
recently,
thisDayInHistory,
thisMonthInHistory,
days,
field,
checkMax=True):
hourField = hourlyFromDailyField[field]
rows = []
plotData = [None] * 25
if field.minValue == 0:
plotData[0] = 0
plotForecast = [None] * 25
today, _ = metarParse.synopticPeriod(datetime.datetime.utcnow())
if dayToPlot is not None:
today = dayToPlot
if type(dayToPlot) is str:
today = datetime.date(*map(int, dayToPlot.split('-')))
snowPerDay = loadHourlyData(city, field, hourField, today)
yesterday = today - datetime.timedelta(days=1)
dailyData = daily.load(city)
forecastData = forecast24Hour.getAndParse(city)
todayUtcMidnight = datetime.datetime(today.year,
today.month,
today.day,
hour=6,
tzinfo=datetime.timezone.utc)
mtz = stations.city[city].timezone
todayUtcMidnightLocal = todayUtcMidnight.astimezone(mtz)
if today in snowPerDay:
if days == 2:
plotData[0] = dailyData[yesterday].TOTAL_SNOW_CM
maxVal, snowHours = snowPerDay[today]
for utctime, vals in forecastData.items():
v = hourField(vals)
if v is None:
continue
timediff = utctime - todayUtcMidnight
index = timediff.days * 24 + timediff.seconds // 3600
if index < 0 or index > 24:
continue
rows.append(('{}h'.format(index), '{}{}'.format(v, field.units)))
plotForecast[index] = {'value': v, 'node': {'r': 6}}
if maxVal is None or v > maxVal:
maxVal = v
for hour, snow in sorted(snowHours.items()):
totalSnow = snow
if days == 2:
totalSnow += dailyData[yesterday].TOTAL_SNOW_CM
utcHour = hour.astimezone(datetime.timezone.utc)
timediff = utcHour - todayUtcMidnight
index = timediff.days * 24 + timediff.seconds // 3600
rows.append(('{}h'.format(hour), '{}{}'.format(snow, field.units)))
plotData[index] = {'value': totalSnow, 'node': {'r': 6}}
print(
tabulate.tabulate(rows,
headers=('Time', hourField.name),
tablefmt="fancy_grid"))
style = pygal.style.Style(label_font_size=15, major_label_font_size=20)
line_chart = pygal.Line(style=style,
print_values=True,
x_label_rotation=80,
include_x_axis=field.minValue == 0)
cityName = stations.city[city].name
line_chart.title = '{cityName} {field.name} on {today:%b %d, %Y}'.format(
**locals())
line_chart.y_title = '{hourField.name} ({field.units})'.format(
**locals())
line_chart.x_labels = map(
lambda t: ('{:%H:%M}'.format(todayUtcMidnightLocal + datetime.
timedelta(hours=t))), range(25))
#print(plotData)
line_chart.add('Observed',
plotData,
stroke_style={'width': 3},
formatter=lambda t: str(t) + field.units)
if plotForecast.count(None) < len(plotForecast):
line_chart.add('Forecast', plotForecast, stroke_style={'width': 3})
historyLines = []
if recently:
historyLines = snowiestDaysHistory(dailyData, today, field)
if thisDayInHistory:
historyLines = maxDaysToday(dailyData,
today,
field,
maxValues=checkMax)
if thisMonthInHistory:
historyLines = maxDaysMonth(dailyData,
today,
field,
maxValues=checkMax)
labelByValue = {}
for history in historyLines:
fv = float(history.snow)
labelByValue[fv] = labelByValue.get(fv, '') + history.label
line_chart.add(history.label, ([{
'value': history.snow,
'node': {
'r': 1
}
}] + [None] * 23 + [history.snow]),
stroke_style={'width': 3},
formatter=lambda t: '')
pprint.PrettyPrinter().pprint(labelByValue)
historyType = ''
if thisDayInHistory:
historyType = 'thisDayInHistory'
if recently:
historyType = 'recently'
if thisMonthInHistory:
historyType = 'thisMonthInHistory'
minMax = 'max' if checkMax else 'min'
fname = '{city}/{today}.hour-{field.name}.{minMax}.{historyType}.png'.format(
**locals())
line_chart.render_to_png(fname, width=1024, height=768)
return fname
#!/usr/bin/python3
# -*- coding: utf-8 -*-
import daily
import datetime
field = daily.SNOW_ON_GRND_CM
data = daily.load('ottawa')
winterStartByYear = {}
winterEndByYear = {}
for year in range(data.minYear, data.maxYear + 1):
daysCount = 0
missingDays = 0
days = daily.dayRange(datetime.date(year, 7, 1),
datetime.date(year + 1, 7, 1))
for day in days:
if day not in data or data[day].SNOW_ON_GRND_FLAG == 'M':
if missingDays > 0:
# Too many days missing, start from scratch
daysCount = 0
missingDays = 0
else:
missingDays += 1
daysCount += 1
continue
if len(data[day].SNOW_ON_GRND_CM) > 0:
depth = int(data[day].SNOW_ON_GRND_CM)
if depth >= 1:
def createPlots(city,
field,
name,
plotDate=None,
plotZeros=True,
verbose=True,
output='svg'):
units = field.units
#
findex = field.index
#
rawData = daily.load(city)
rawForecast = forecast.getForecastDataEnvCan(city)
#
if verbose:
print('annualData(..., field = %s, plotDate = %s)' % (field, plotDate))
#
if plotDate == None:
# if no date was provided, use the last date for which data is available
plotDate = max(rawData.keys())
print("No plot-date was provided, using {}".format(plotDate))
#
for (date, val) in rawForecast.items():
print(date, val)
mergedData = daily.Data(rawData.copy())
for k, v in rawForecast.items():
if k not in mergedData:
mergedData[k] = v
plotData = annualData(rawData=mergedData, field=field, plotDate=plotDate)
toDel = []
for year in plotData:
value = plotData[year]
if value == None:
toDel.append(year)
if plotZeros == False and value == 0:
# We've been told to skip zeros, so we don't plot them
toDel.append(year)
for year in toDel:
del plotData[year]
del toDel
#culledPlotData = makefit.makeFit(plotData, 75)
plotLine, plotTop10, plotBottom10 = createPlot(
plotData, city, name, '%s (%s)' % (field.englishName, units),
field.englishName, units, plotDate, plotZeros, verbose)
returnValue = plotLine, plotTop10, plotBottom10
yearByValue = {}
valueByYear = {}
recentValues = []
for year in plotData:
#
value = plotData[year]
#if value == None:
# value = 0
if (value not in yearByValue):
yearByValue[value] = [year]
else:
yearByValue[value].append(year)
valueByYear[year] = value
if year < plotDate.year and year > plotDate.year - 31:
recentValues.append(value)
#
if verbose:
avg = sum(recentValues) / len(recentValues)
print("Average is {:.1f}{}".format(float(avg), units))
#
currentVariance = "unknown"
if plotDate.year in valueByYear:
thisYearValue = valueByYear[plotDate.year]
if plotDate.year in valueByYear:
currentVariance = "%.1f%s %s average" % (
abs(thisYearValue - avg), units, ["below", "above"
][thisYearValue > avg])
print("{} is {}".format(plotDate.year, currentVariance))
print([
float(v)
for v in filter(lambda v: v < thisYearValue, recentValues)
])
print([
float(v)
for v in filter(lambda v: v == thisYearValue, recentValues)
])
print([
float(v)
for v in filter(lambda v: v > thisYearValue, recentValues)
])
#
return returnValue
#!/usr/bin/python3
# -*- coding: utf-8 -*-
import daily, sys, datetime
from collections import namedtuple
from namedList import *
from fieldOperators import *
now = datetime.datetime.now().date()
weatherData = daily.load('ottawa')
SumCount = namedList('SumCount', ('sum', 'count'))
ValueWithYearA = namedStruct('ValueWithYear', 'value year')
class ValueWithYear(ValueWithYearA):
def __lt__(self, other):
return self.value < other.value
DistrobutionCount = namedStruct('DistrobutionCount', ('sum=0', 'count=0', 'average=None', 'min=None', 'max=None'))
def JustJanuary(day):
return day.month == 1
class JustAMonth:
def __init__(self, month):
self.month = month
def __call__(self, day):
return self.month == day.month
def BeforeToday(day):
return day.month < now.month or ( day.month == now.month and day.day < now.day)
#!/usr/bin/python
# -*- coding: utf-8 -*-
from __future__ import print_function
import daily, time
data = daily.load("farm")
records = {}
consequtiveRecords = 0
def reportRecords(dateTime):
global consequtiveRecords
if consequtiveRecords >= 3:
print dateTime, consequtiveRecords
consequtiveRecords = 0
for dateTime in sorted(data.keys()):
if len(data[dateTime].MAX_TEMP) > 0:
v = float(data[dateTime].MAX_TEMP)
monthDay = (dateTime.month, dateTime.day)
if monthDay not in records:
records[monthDay] = v
reportRecords(dateTime)
elif records[monthDay] <= v:
#print "Record %d %s %.1fC" % (dateTime.year, monthDay, v)
records[monthDay] = v
consequtiveRecords += 1
else:
