def load_notes(weather_notes, file_name, weather_data):
with open(file_name, 'r') as f:
id = 1
with weather_notes.open() as collection:
with weather_data.open() as data:
while True:
line = f.readline()
if len(line) == 0:
break
elif line[0] == '#':
continue
note_words = line.split()
if note_words:
note = collection.new_entity()
note['ID'].set_from_value(id)
start = iso.TimePoint(date=iso.Date.from_str(
note_words[0]),
time=iso.Time(hour=0,
minute=0,
second=0))
note['StartDate'].set_from_value(start)
end = iso.TimePoint(date=iso.Date.from_str(
note_words[1]).offset(days=1),
time=iso.Time(hour=0,
minute=0,
second=0))
note['EndDate'].set_from_value(end)
note['Details'].set_from_value(
string.join(note_words[2:], ' '))
collection.insert_entity(note)
# now find the data points that match
data.set_filter(
core.CommonExpression.from_str(
"TimePoint ge datetime'%s' and "
"TimePoint lt datetime'%s'" %
(unicode(start), unicode(end))))
for data_point in data.values():
# use values, not itervalues to avoid this bug
# in Python 2.7 http://bugs.python.org/issue10513
data_point['Note'].bind_entity(note)
data.update_entity(data_point)
id = id + 1
with weather_notes.open() as collection:
collection.set_orderby(
core.CommonExpression.orderby_from_str('StartDate desc'))
for e in collection.itervalues():
with e['DataPoints'].open() as affectedData:
print "%s-%s: %s (%i data points affected)" % (
unicode(e['StartDate'].value), unicode(e['EndDate'].value),
e['Details'].value, len(affectedData))
def LoadNotes(weatherNotes, fileName, weatherData):
with open(fileName, 'r') as f:
id = 1
with weatherNotes.OpenCollection() as collection, weatherData.OpenCollection() as data:
while True:
line = f.readline()
if len(line) == 0:
break
elif line[0] == '#':
continue
noteWords = line.split()
if noteWords:
note = collection.NewEntity()
note['ID'].SetFromValue(id)
start = iso.TimePoint(
date=iso.Date.FromString(noteWords[0]),
time=iso.Time(hour=0, minute=0, second=0))
note['StartDate'].SetFromValue(start)
end = iso.TimePoint(
date=iso.Date.FromString(noteWords[1]).Offset(days=1),
time=iso.Time(hour=0, minute=0, second=0))
note['EndDate'].SetFromValue(end)
note['Details'].SetFromValue(
string.join(noteWords[2:], ' '))
collection.InsertEntity(note)
# now find the data points that match
data.Filter(
core.CommonExpression.FromString(
"TimePoint ge datetime'%s' and TimePoint lt datetime'%s'" %
(unicode(start), unicode(end))))
for dataPoint in data.values():
# use values, not itervalues to avoid this bug in Python 2.7
# http://bugs.python.org/issue10513
dataPoint['Note'].BindEntity(note)
data.UpdateEntity(dataPoint)
id = id + 1
with weatherNotes.OpenCollection() as collection:
collection.OrderBy(
core.CommonExpression.OrderByFromString('StartDate desc'))
for e in collection.itervalues():
with e['DataPoints'].OpenCollection() as affectedData:
print "%s-%s: %s (%i data points affected)" % (
unicode(e['StartDate'].value), unicode(e['EndDate'].value),
e['Details'].value, len(affectedData))
class MockTime(object):
#: the float time (since the epoch) corresponding to 01 January
#: 1970 00:00:00 UTC the OAuth time origin.
oauth_origin = float(
iso.TimePoint(date=iso.Date(century=19, year=70, month=1, day=1),
time=iso.Time(hour=0, minute=0, second=0,
zdirection=0)).get_unixtime())
def __init__(self, base=None):
if base is None:
self.now = time.time()
else:
self.now = base
def time(self):
return self.now
def tick(self, delta=1.0):
self.now += delta
def RequireFullDate(self):
"""Parses a :py:class:`FullDate` instance raising
:py:class:`SyntaxError` if none is found.
There are three supported formats as described in the specification::
"Sun, 06 Nov 1994 08:49:37 GMT"
"Sunday, 06-Nov-94 08:49:37 GMT"
"Sun Nov 6 08:49:37 1994"
The first of these is the preferred format."""
century = None
year = None
month = None
day = None
hour = None
minute = None
second = None
dayOfWeek = None
self.ParseSP()
token = self.RequireToken("day-of-week").lower()
dayOfWeek = self._wkdayTable.get(token, None)
if dayOfWeek is None:
raise SyntaxError("Unrecognized day of week: %s" % token)
self.ParseSP()
if self.ParseSeparator(","):
self.ParseSP()
if self.IsToken() and IsDIGITS(self.cWord):
# Best format 0: "Sun, 06 Nov 1994 08:49:37 GMT" - the
# preferred format!
day = self.RequireInteger("date")
self.ParseSP()
token = self.RequireToken("month").lower()
month = self._monthTable.get(token, None)
if month is None:
raise SyntaxError("Unrecognized month: %s" % repr(token))
self.ParseSP()
year = self.RequireInteger("year")
century = year // 100
year = year % 100
else:
# Alternative 1: "Sunday, 06-Nov-94 08:49:37 GMT"
token = self.RequireToken("DD-MMM-YY")
sToken = token.split('-')
if len(sToken) != 3 or not IsDIGITS(sToken[0]) or not IsDIGITS(
sToken[2]):
raise SyntaxError("Expected DD-MMM-YY, found %s" %
repr(token))
day = int(sToken[0])
year = int(sToken[2])
month = self._monthTable.get(sToken[1].lower(), None)
if month is None:
raise SyntaxError("Unrecognized month: %s" %
repr(sToken[1]))
else:
# "Sun Nov 6 08:49:37 1994"
token = self.RequireToken("month").lower()
month = self._monthTable.get(token, None)
if month is None:
raise SyntaxError("Unrecognized month: %s" % repr(token))
self.ParseSP()
day = self.RequireInteger("date")
self.ParseSP()
hour = self.RequireInteger("hour")
self.RequireSeparator(':')
minute = self.RequireInteger("minute")
self.RequireSeparator(':')
second = self.RequireInteger("second")
self.ParseSP()
if year is None:
year = self.RequireInteger("year")
century = year // 100
year = year % 100
else:
token = self.RequireToken("GMT").upper()
if token != "GMT":
raise SyntaxError("Unrecognized timezone: %s" % repr(token))
if century is None:
if year < 90:
century = 20
else:
century = 19
tp = FullDate(date=iso.Date(century=century,
year=year,
month=month + 1,
day=day),
time=iso.Time(hour=hour,
minute=minute,
second=second,
zDirection=0))
d1, d2, d3, d4, dow = tp.date.GetWeekDay()
if dow != dayOfWeek + 1:
raise SyntaxError(
"Day-of-week mismatch, expected %s but found %s" %
(self.wkday[dow - 1], self.wkday[dayOfWeek]))
return tp
def LoadDataFromFile(weatherData, f, year, month, day):
with weatherData.OpenCollection() as collection:
while True:
line = f.readline()
if len(line) == 0:
break
elif line[0] == '#':
continue
data = line.split()
if not data:
continue
if len(data) < 11:
data = data + ['*'] * (11 - len(data))
for i in (1, 3, 5, 7, 8, 10):
try:
data[i] = float(data[i])
except ValueError:
data[i] = None
for i in (2, 4, 6):
try:
data[i] = int(data[i])
except ValueError:
data[i] = None
dataPoint = collection.new_entity()
hour, min = map(int, data[0].split(':'))
tValue = iso.TimePoint(date=iso.Date(century=year / 100,
year=year % 100,
month=month,
day=day),
time=iso.Time(hour=hour,
minute=min,
second=0))
bst = IsBST(tValue)
if bst is not False:
# assume BST for now, add the zone info and then shift to GMT
tValue = tValue.WithZone(zDirection=1,
zHour=1).ShiftZone(zDirection=0)
dataPoint['TimePoint'].SetFromValue(tValue)
dataPoint['Temperature'].SetFromValue(data[1])
dataPoint['Humidity'].SetFromValue(data[2])
dataPoint['DewPoint'].SetFromValue(data[3])
dataPoint['Pressure'].SetFromValue(data[4])
dataPoint['WindSpeed'].SetFromValue(data[5])
dataPoint['WindDirection'].SetFromValue(data[6])
dataPoint['Sun'].SetFromValue(data[7])
dataPoint['Rain'].SetFromValue(data[8])
shour, smin = map(int, data[9].split(':'))
dataPoint['SunRainStart'].SetFromValue(
iso.Time(hour=shour, minute=smin, second=0))
dataPoint['WindSpeedMax'].SetFromValue(data[10])
try:
collection.insert_entity(dataPoint)
except edm.ConstraintError:
if bst is None:
# This was an ambiguous entry, the first one is in BST, the
# second one is in GMT as the clocks have gone back, so we
# shift forward again and then force the zone to GMT
tValue = tValue.ShiftZone(zDirection=1,
zHour=1).WithZone(zDirection=0)
dataPoint['TimePoint'].SetFromValue(tValue)
logging.info("Auto-detecting switch to GMT at: %s",
str(tValue))
try:
collection.insert_entity(dataPoint)
except KeyError:
logging.error(
"Duplicate data point during BST/GMT switching: %s",
str(tValue))
else:
logging.error("Unexpected duplicate data point: %s",
str(tValue))
def load_data_from_file(weather_data, f, year, month, day):
with weather_data.open() as collection:
while True:
line = f.readline().decode('ascii')
if len(line) == 0 or line.startswith('Date unknown.'):
break
elif line[0] == '#':
continue
data = line.split()
if not data:
continue
if len(data) < 11:
data = data + ['*'] * (11 - len(data))
for i in (1, 3, 5, 7, 8, 10):
try:
data[i] = float(data[i])
except ValueError:
data[i] = None
for i in (2, 4):
try:
data[i] = int(data[i])
except ValueError:
data[i] = None
data[6] = data[6].strip()
data_point = collection.new_entity()
hour, min = [int(i) for i in data[0].split(':')]
tvalue = iso.TimePoint(
date=iso.Date(century=year // 100, year=year %
100, month=month, day=day),
time=iso.Time(hour=hour, minute=min, second=0))
bst = is_bst(tvalue)
if bst is not False:
# assume BST for now, add the zone info and then shift to GMT
tvalue = tvalue.with_zone(
zdirection=1, zhour=1).shift_zone(zdirection=0)
data_point['TimePoint'].set_from_value(tvalue)
data_point['Temperature'].set_from_value(data[1])
data_point['Humidity'].set_from_value(data[2])
data_point['DewPoint'].set_from_value(data[3])
data_point['Pressure'].set_from_value(data[4])
data_point['WindSpeed'].set_from_value(data[5])
data_point['WindDirection'].set_from_value(data[6])
data_point['Sun'].set_from_value(data[7])
data_point['Rain'].set_from_value(data[8])
shour, smin = [int(i) for i in data[9].split(':')]
data_point['SunRainStart'].set_from_value(
iso.Time(hour=shour, minute=smin, second=0))
data_point['WindSpeedMax'].set_from_value(data[10])
try:
collection.insert_entity(data_point)
except edm.ConstraintError:
if bst is None:
# This was an ambiguous entry, the first one is in
# BST, the second one is in GMT as the clocks have
# gone back, so we shift forward again and then
# force the zone to GMT
tvalue = tvalue.shift_zone(
zdirection=1, zhour=1).with_zone(zdirection=0)
data_point['TimePoint'].set_from_value(tvalue)
logging.info(
"Auto-detecting switch to GMT at: %s", str(tvalue))
try:
collection.insert_entity(data_point)
except KeyError:
logging.error("Duplicate data point during BST/GMT "
"switching: %s", str(tvalue))
else:
logging.error(
"Unexpected duplicate data point: %s", str(tvalue))