def consecutiveHours(city, expr):
data = hourly.load(city)
tz = stations.city[city].timezone
runs = {}
allRuns = []
curRun = []
mostRecentRun = None
#
timestamps = sorted(data.keys())
#
for utc in hourrange(timestamps[0], timestamps[-1]):
hourdata = data.get(utc)
ltime = utc.astimezone(tz)
v = None
if hourdata is not None:
v = hourdata.eval(expr, {'time':ltime} )
if v is not None and v is not False:
curRun.append((ltime, v))
else:
r = addTo(runs, allRuns, curRun)
if r is not None:
mostRecentRun = r
curRun = []
r = addTo(runs, allRuns, curRun)
if r is not None:
mostRecentRun = r
return runs, allRuns, mostRecentRun
def loadHourlyData(city, dayField, hourField, day):
if dayField == daily.TOTAL_SNOW_CM:
return metarParse.loadSnowWithHours(city)
elif dayField in (daily.MAX_TEMP, daily.MIN_TEMP):
return loadHourlyDataSwob(city, dayField, day)
mytimezone = stations.city[city].timezone
cityData = daily.load(city)
utcDayStart = datetime.datetime(day.year,
day.month,
day.day,
6,
tzinfo=datetime.timezone.utc)
dayStart = utcDayStart.astimezone(mytimezone)
dayEnd = dayStart + datetime.timedelta(days=1)
cityHourData = hourly.load(city, dateRange=(dayStart, dayEnd))
valByHour = {}
for utcHour, values in cityHourData.items():
v = hourField(values)
if v is None:
continue
l = utcHour.astimezone(mytimezone)
valByHour[l] = v
return {
day: metarParse.SnowAndHourly(cityData[day][dayField.index], valByHour)
}
def loadHourlyDataSwob(city, dayField, day):
if dayField == daily.TOTAL_SNOW_CM:
return metarParse.loadSnowWithHours(city)
mytimezone = stations.city[city].timezone
cityData = daily.load(city)
utcDayStart = datetime.datetime(day.year,
day.month,
day.day,
6,
tzinfo=datetime.timezone.utc)
dayStart = utcDayStart.astimezone(mytimezone)
dayEnd = dayStart + datetime.timedelta(days=1)
cityHourData = gatherSwob.parse(city)
hourly.load(city, dateRange=(dayStart, dayEnd))
valByHour = {}
for valuesDict in cityHourData:
ts = valuesDict['time']
if ts < dayStart or ts >= dayEnd:
continue
if ts == utcDayStart:
v = valuesDict['air_temp']
elif dayField == daily.MAX_TEMP:
v = valuesDict['max_air_temp_pst1hr']
elif dayField == daily.MIN_TEMP:
v = valuesDict['min_air_temp_pst1hr']
else:
assert (False)
if v is None:
continue
v = D(v)
l = ts.astimezone(mytimezone)
valByHour[l] = v
dayData = cityData.get(day, None)
if dayData is not None:
return {
day: metarParse.SnowAndHourly(dayData[dayField.index], valByHour)
}
return {day: metarParse.SnowAndHourly(None, valByHour)}
def dispatch(cityName,
firstYear,
startMonth,
startDay,
endMonth,
endDay,
expression,
verbose=True,
skipIncomplete=False,
lineChart=False):
data = hourly.load(cityName)
global START_MONTH, START_DAY, END_MONTH, END_DAY
START_MONTH = startMonth
START_DAY = startDay
END_MONTH = endMonth
END_DAY = endDay
checkList = {
'>=30': [
GreaterThanOrEqualToWithFlag(hourly.TEMP, 30),
"hours above or at 30"
],
'<10': [LessThanOrEqualToWithFlag(hourly.TEMP, 10), "hours below 10"],
'<=-20':
[LessThanOrEqualToWithFlag(hourly.TEMP, -20), "hours below or at -20"],
}
args = checkList[expression]
return count(cityName,
data,
args[0],
args[1],
verbose,
skipIncomplete=skipIncomplete,
lineChart=lineChart)
text = ("{} between {}➜{},"
" the temp dropped from {:.1f}℃➜{:.1f}℃"
" which was #{}'s largest 1-hour drop {}"
.format(dayName,
clock12(city, 'H{}'.format(maxDiffD[0].hour)),
clock12(city, 'H{}'.format(maxDiffD[1].hour)),
float(maxDiffValue[0]),
float(maxDiffValue[1]),
stations.city[city].name,
since))
(use, tweet) = shouldTweetSuffix(
city, text)
if use:
delayedTweets.addToListForCity(city, tweet, urgent=True)
print('ok')
else:
print('skipping')
break
maxDiff = diff
maxDiffValue = v, lastV
maxDiffD = localTime, lastD
lastV = v
lastD = localTime
lastUtcTime = utcTime
if __name__ == '__main__':
data = hourly.load('ottawa')
checkMaxDrop2('ottawa', 'maxDrop', recentLimit=1)
# -*- coding: utf-8 -*-
import hourly, sys, copy, fnmatch
import argparse, datetime
import stations
parser = argparse.ArgumentParser(
description='Determine the last time a field has been this high/low.')
parser.add_argument('-f', help='Field')
parser.add_argument('--city', default='ottawa')
parser.add_argument('-d', help='Mask', nargs='*')
args = parser.parse_args()
field = args.f
data = hourly.load(args.city)
days = args.d
tz = stations.city[args.city].timezone
for d in days:
dayStart = datetime.datetime.strptime(d, "%Y-%m-%d").date()
print('---', dayStart)
for utc in sorted(data.keys()):
ltime = utc.astimezone(tz)
if ltime.date() != dayStart:
continue
if utc in data:
windchill = data[utc].windchill
if windchill is not None:
windchill = '{:.1f}'.format(windchill)
#!/usr/bin/python
from __future__ import print_function
import hourly
data = hourly.load("ottawa")
hours = []
for hour, hourdata in data.iteritems():
visibility = hourdata[hourly.VISIBILITY]
if len(visibility):
visibility = float(visibility)
#if visibility <= 0.2 and 'fog' in hourdata[hourly.WEATHER].lower():
if visibility <= 1 and 'fog' in hourdata[hourly.WEATHER].lower():
hours.append(hour)
#print hour
hours.sort()
for hour in hours:
print hour
exit(0)
days = {}
for hour in hours:
day = hour[:-1]
if day not in days:
days[day] = 0
days[day] += 1
def main(city, year):
data = hourly.load(
city,
(datetime.datetime(year, 1, 1), datetime.datetime(year + 1, 1, 1)))
statsByDay = {}
for dateTime in sorted(data.keys()):
info = data[dateTime]
localTime = dateTime.astimezone(stations.city[city].timezone)
date = localTime.date()
#if (datetime.date.today() - date).days < 3:
#print(dateTime, info)
try:
thisDayStats = statsByDay[date]
except KeyError:
statsByDay[date] = {}
thisDayStats = statsByDay[date]
for hourFieldName, val in info._asdict().items():
fieldName = nameMap[hourFieldName]
if val is None or (type(val) is str and len(val) == 0):
# Skip empty fields and textual weather field
continue
try:
thisDayStats[fieldName][localTime.hour] = val
except KeyError:
thisDayStats[fieldName] = {localTime.hour: val}
#print('hourFieldName, val,', repr(hourFieldName), repr(val), repr(thisDayStats[hourFieldName]))
val = info.humidex
if val != None:
fname = 'Humidex'
try:
thisDayStats[fname][localTime.hour] = val
except KeyError:
thisDayStats[fname] = {localTime.hour: val}
val = info.windchill
if val != None:
fname = 'Windchill'
try:
thisDayStats[fname][localTime.hour] = val
except KeyError:
thisDayStats[fname] = {localTime.hour: val}
fname = "{city}/data/{year}.dailyextra-csv.bz2".format(**locals())
#print(fname)
csvWriter = csv.writer(io.TextIOWrapper(bz2.BZ2File(fname, 'w')))
flagMap = {'Humidex': 'Temp', 'Windchill': 'Temp'}
for date in sorted(statsByDay.keys()):
infodict = statsByDay[date]
#print(date, infodict)
info = {}
if date == envCanToday():
_, _, summaryMaxGust = todayXmlParse.getMinMax(city)
info['maxGust'] = summaryMaxGust
info['maxGustFlag'] = 'H'
for maxName in ('Temp', 'Humidex'):
maxVal = None
maxFlag = ''
lenVals = 0
if maxName in infodict:
valByHour = infodict[maxName]
lenVals = len(valByHour)
minKey, maxKey = keyOfMinMaxFloatValue(valByHour)
if maxKey is not None:
maxHourlyHour = maxKey[0]
maxHourlyValue = valByHour[maxHourlyHour]
maxVal = maxHourlyValue
maxFlag = 'H' + str(maxHourlyHour)
info['max' + maxName] = maxVal
#print(infodict)
flagName = flagMap.get(maxName, maxName) + 'Flag'
try:
flagSet = set(infodict[flagName].values())
except KeyError:
flagSet = set()
if len(maxFlag) > 0:
# MAX_Temp should always be marked as HOURLY to avoid
# confusion with official daytime highs
if 'H' in flagSet:
flagSet.remove('H')
flagSet.add(maxFlag)
if 'M' in flagSet and len(flagSet) > 1:
flagSet.remove('M')
if lenVals > 0 and lenVals < hourCount(city, date):
flagSet.add('I')
info['max' + maxName + 'Flag'] = '+'.join(sorted(flagSet))
for minName in ('Temp', 'Windchill'):
#if minName == 'Windchill' and date == datetime.date(1972,1,26):
# import pudb; pu.db
minVal = None
minFlag = ''
lenVals = 0
if minName in infodict:
valByHour = infodict[minName]
lenVals = len(valByHour)
minKey, maxKey = keyOfMinMaxFloatValue(valByHour)
if minKey is not None:
minHourlyHour = minKey[0]
minHourlyValue = valByHour[minHourlyHour]
minVal = minHourlyValue
minFlag = 'H' + str(minHourlyHour)
flagName = flagMap.get(minName, minName) + 'Flag'
try:
flagSet = set(infodict[flagName].values())
except KeyError:
flagSet = set()
if len(minFlag) > 0:
# minTemp should always be marked as HOURLY to avoid
# confusion with official daytime lows
if 'H' in flagSet:
flagSet.remove('H')
flagSet.add(minFlag)
if 'M' in flagSet and len(flagSet) > 1:
flagSet.remove('M')
if lenVals > 0 and lenVals < hourCount(city, date):
flagSet.add('I')
info['min' + minName] = minVal
info['min' + minName + 'Flag'] = '+'.join(sorted(flagSet))
for avgName in ('Temp', 'Windchill', 'Wind', 'Humidity', 'Dewpoint'):
avgVal = None
lenVals = 0
if avgName in infodict:
valByHour = infodict[avgName].values()
floatVals = tuple(filter(lambda t: t is not None, valByHour))
if len(floatVals) > 0:
avgVal = numpy.average(tuple(map(float, floatVals)))
if floatVals[0] is int:
avgVal = int(avgVal)
else:
avgVal = D(avgVal).quantize(floatVals[0],
decimal.ROUND_HALF_UP)
lenVals = len(floatVals)
flagName = flagMap.get(avgName, avgName) + 'Flag'
try:
flagSet = set(infodict[flagName].values())
except KeyError:
flagSet = set()
if 'M' in flagSet and avgVal is not None:
flagSet.remove('M')
#print(date, avgName, lenVals)
if lenVals > 0 and lenVals < hourCount(city, date):
flagSet.add('I')
# All averages should always be marked as HOURLY to avoid
# confusion with official MEAN_Temp
if avgVal is not None:
flagSet.add('H')
info['avg' + avgName] = avgVal
info['avg' + avgName + 'Flag'] = '+'.join(sorted(flagSet))
info = HourlyDailyData(**info)
intInfo = prepForDb(info)
csvWriter.writerow((date.year, date.month, date.day) + intInfo)
if (datetime.date.today() - date).days < 3:
print(date, intInfo)
def main():
HISTORY = 10
parser = argparse.ArgumentParser(
description='Determine how often some weather occurs.')
parser.add_argument('expr', help='Which observation to check')
parser.add_argument('--city', default='ottawa')
parser.add_argument('--run', type=int, default=1)
parser.add_argument('-m', help='Mask', default=['*'], nargs='*')
parser.add_argument(
'--between',
help=
'Only consider dates between these two. Comma separated like 05-15,07-01'
)
parser.add_argument('--holiday', help='The name of a holiday')
parser.add_argument('--hour',
help='Use hourly data instead of daily.',
action='store_true',
default=False)
parser.add_argument('--winters',
help='Count by winter instead of by year.',
action='store_true',
default=False)
parser.add_argument('--group-by-year', action='store_true', default=False)
args = parser.parse_args()
run = args.run
city = args.city
fieldDict = {
'min': Value(daily.MIN_TEMP),
'max': Value(daily.MAX_TEMP),
'meanTemp': Value(daily.MEAN_TEMP),
'tempSpan': ValueDiff(daily.MAX_TEMP, daily.MIN_TEMP),
'rain': Value(daily.TOTAL_RAIN_MM),
'precip': Value(daily.TOTAL_PRECIP_MM),
'humidex': Value(daily.MAX_HUMIDEX),
'snow': Value(daily.TOTAL_SNOW_CM),
'snowpack': ValueEmptyZero(daily.SNOW_ON_GRND_CM),
'windgust': Value(daily.SPD_OF_MAX_GUST_KPH),
'wind': Value(daily.AVG_WIND),
'windchill': Value(daily.MIN_WINDCHILL),
'avgWindchill': Value(daily.MIN_WINDCHILL),
}
if args.hour:
fieldDict = {
'temp': Value(hourly.TEMP),
'dewpoint': Value(hourly.DEW_POINT_TEMP),
'humidity': Value(hourly.REL_HUM),
'windDir': Value(hourly.WIND_DIR),
'wind': Value(hourly.WIND_SPD),
'visibility': Value(hourly.VISIBILITY),
'pressure': Value(hourly.STN_PRESS),
'weather': ValueNoFlag(hourly.WEATHER),
'windchill': ValueNoFlag(hourly.WINDCHILL),
}
def allFieldNames():
return fieldDict.keys()
dateFilter = args.m
between = parseBetween(args.between)
if args.hour:
data = hourly.load(city)
for time, conditions in parseWeatherStatsRealtime.parse(city).items():
if time.minute == 0:
values = data.get(time, None)
if values is None:
data[time] = hourly.HourData(WEATHER=conditions)
else:
data[time] = values._replace(WEATHER=conditions)
for time, metarWeather in metarParse.genHourlyWeather(city):
if time.minute != 0:
continue
#if time.date() == dt.date(2017,3,27):
# import pudb; pu.db
values = data.get(time, None)
if values is None:
data[time] = hourly.HourData(WEATHER=metarWeather)
elif (values.WEATHER is None or values.WEATHER == 'NA'):
data[time] = values._replace(WEATHER=metarWeather)
else:
data = daily.load(city)
maxVal = None
curRun = deque()
curDates = deque()
fieldValues = fieldDict.values()
firstDate = None
matches = []
referencedValues = set()
compiledExpr = compile(args.expr, filename='.', mode='eval')
exprWords = pythonWords(args.expr)
for fieldName in fieldDict.keys():
if fieldName in exprWords:
referencedValues.add(fieldName)
#print(referencedValues)
#print(tuple(allFieldNames()))
class refCountedList(deque):
def __init__(self):
self.indexSet = set()
def __getitem__(self, ind):
if type(ind) != slice: #print('[{ind}]'.format(**locals()))
self.indexSet.add(ind)
return deque.__getitem__(self, ind)
return list(self)[ind]
def clearIndexSet(self):
self.indexSet.clear()
historyDates = deque([])
history = refCountedList()
class History():
pass
expectedDiff = dt.timedelta(days=1)
if args.hour:
expectedDiff = dt.timedelta(hours=1)
mytimezone = stations.city[args.city].timezone
for date in sorted(data.keys()):
if args.hour:
utchour = date
localhour = utchour.astimezone(mytimezone)
date = localhour
if (len(historyDates) > 0 and date - historyDates[0] != expectedDiff):
historyDates.clear()
history.clear()
if matchDate(date, args.m, between, args.holiday):
#if date.year == 2016 and date.month == 11 and date.day == 30:
# import pudb; pu.db
vals = {
'history': history,
"__builtins__": __builtins__,
'time': date
}
history.appendleft(History())
historyDates.appendleft(date)
for fieldName, fieldCall in fieldDict.items():
vals[fieldName] = fieldCall(data[date], date)
flagValue = fieldCall.getFlag(data[date])
if flagValue is not None:
vals[fieldName + 'Flag'] = flagValue
if type(vals[fieldName]) is not SpecialNone:
history[0].__setattr__(fieldName, vals[fieldName])
if flagValue is not None:
history[0].__setattr__(fieldName + 'Flag', flagValue)
skip = False
usedVals = {}
for fieldName in referencedValues:
if type(vals[fieldName]) is SpecialNone:
#print('Skipping {} because {} is None'.format(date, fieldName))
skip = True
break
#print(fieldName, type(vals[fieldName]) is SpecialNone)
usedVals[fieldName] = vals[fieldName]
if skip:
continue
if firstDate is None:
firstDate = date
lastDate = date
#expr = args.expr.format(**vals)
#print(date, args.expr, usedVals, vals)
history.clearIndexSet()
try:
val = eval(compiledExpr, vals)
except IndexError:
val = False
except AttributeError:
val = False
except TypeError:
for offset, date in enumerate(historyDates):
print(date)
for name in allFieldNames():
if hasattr(history[offset], name):
print(name, history[offset].__getattribute__(name))
raise
#print(val)
if val is True or type(val) is tuple and val[0] is True:
#print('+++')
#for offset, date in enumerate(historyDates):
# print(date)
# for name in allFieldNames():
# if name in dir(history[offset]):
# print(name, history[offset].__getattribute__(name))
#print('---')
for i in history.indexSet:
for fieldName in referencedValues:
usedVals['history[{}].{}'.format(i, fieldName)] = (
history[i].__getattribute__(fieldName))
#print(history.indexSet)
expectedDate = date
if len(curDates) > 0:
expectedDate = curDates[-1] + expectedDiff
if date != expectedDate:
#print('Clearing run', date, curDates[-1])
curDates = deque([date])
curRun = deque([usedVals])
if type(val) is tuple:
curRun = deque([{'expr': val[1]}])
else:
curDates.append(date)
if type(val) is tuple:
curRun.append({'expr': val[1]})
else:
curRun.append(usedVals)
if len(curRun) > run:
curRun.popleft()
curDates.popleft()
#print(date, len(curRun), run)
if len(curRun) == run:
printDate(curDates)
if len(curRun[0]) == 1:
print(' '.join(
[str(first(a.values())) for a in curRun]))
else:
print(curRun)
matches.append(curDates)
if args.winters:
grouping = tuple([winterFromDate(a[0]) for a in matches])
group = 'winter'
else:
grouping = tuple([a[0].year for a in matches])
group = 'year'
if args.group_by_year:
print(tuple(enumerate(Counter(grouping).most_common(30))))
print('Total count: {}'.format(len(grouping)))
print('Occurance: ', end='')
yearSpan = lastDate.year - firstDate.year + 1
if len(matches) < yearSpan:
print('once every {:.1f} {}s'.format(yearSpan / len(matches), group))
else:
print('{:.1f} times per {}'.format(len(matches) / yearSpan, group))
print('Yearly Occurance:', end=' ')
uniqGrouping = sorted(list(set(grouping)))
if len(uniqGrouping) < yearSpan:
occurance = yearSpan / len(uniqGrouping)
print('once every {occurance:.1f} {group}s,'.format(**locals()),
end=' ')
else:
occurance = len(uniqGrouping) / yearSpan
print('{occurance:.1f} times per {group},'.format(**locals()), end=' ')
recentFilter = lambda t: t >= today().year - 30 and t < today().year
if args.winters:
recentFilter = lambda t: t >= thisWinter() - 30 and t < thisWinter()
recentOccursYears = tuple(filter(recentFilter, grouping))
recentUniqYears = tuple(sorted(set(recentOccursYears)))
print('{} out of the past 30 years: {}'.format(len(recentUniqYears),
str(recentUniqYears)))
recentYears = range(thisYear() - 30, thisYear())
if args.winters:
recentYears = range(thisWinter() - 30, thisWinter())
recentLen = len(recentUniqYears)
recentYearCounter = Counter(recentOccursYears)
recentLens = sorted(recentYearCounter[a] for a in recentYears)
print('{} during the past 30 years'.format(recentLen))
if True: #recentLen > 30:
print('Average is {:.1f}/year during the past 30 years'.format(
len(recentOccursYears) / 30))
print('Median is {},{}/year during the past 30 years'.format(
recentLens[int(len(recentLens) / 2)], recentLens[int(
(len(recentLens) + 1) / 2)]))
print(
'80% CI is {},{}'.format(
recentLens[len(recentLens) // 10],
recentLens[len(recentLens) - (len(recentLens) // 10) - 1]),
recentLens)
#print(sorted([(len(o),y) for y,o in recentByYear.items()]))
else:
print('Every {:.1f} years during the past 30 years'.format(30 /
recentLen))
occurDays = '∞'
if recentLen > 0:
occurDays = (dt.date.today() - yearsAgo(30)).days / recentLen
print('Every {} days during the past 30 years'.format(occurDays))
if recentLen > 0:
print('Every {} months, {:.1f} days during the past 30 years'.format(
int(occurDays // 30), occurDays % 30))
if args.hour:
recentOccur = tuple(
filter(lambda t: t[0].date() >= yearsAgo(1), matches))
else:
recentOccur = tuple(filter(lambda t: t[0] >= yearsAgo(1), matches))
print('{} during the past year'.format(len(recentOccur)))
if args.winters:
thisWinterCount = grouping.count(thisWinter())
print('{} so far this winter'.format(thisWinterCount))
else:
thisYearCount = grouping.count(thisYear())
print('{} so far this year'.format(thisYearCount))
