def load_notes(weather_notes, file_name, weather_data):
with open(file_name, 'r') as f:
id = 1
with weather_notes.open() as collection:
with weather_data.open() as data:
while True:
line = f.readline()
if len(line) == 0:
break
elif line[0] == '#':
continue
note_words = line.split()
if note_words:
note = collection.new_entity()
note['ID'].set_from_value(id)
start = iso.TimePoint(date=iso.Date.from_str(
note_words[0]),
time=iso.Time(hour=0,
minute=0,
second=0))
note['StartDate'].set_from_value(start)
end = iso.TimePoint(date=iso.Date.from_str(
note_words[1]).offset(days=1),
time=iso.Time(hour=0,
minute=0,
second=0))
note['EndDate'].set_from_value(end)
note['Details'].set_from_value(
string.join(note_words[2:], ' '))
collection.insert_entity(note)
# now find the data points that match
data.set_filter(
core.CommonExpression.from_str(
"TimePoint ge datetime'%s' and "
"TimePoint lt datetime'%s'" %
(unicode(start), unicode(end))))
for data_point in data.values():
# use values, not itervalues to avoid this bug
# in Python 2.7 http://bugs.python.org/issue10513
data_point['Note'].bind_entity(note)
data.update_entity(data_point)
id = id + 1
with weather_notes.open() as collection:
collection.set_orderby(
core.CommonExpression.orderby_from_str('StartDate desc'))
for e in collection.itervalues():
with e['DataPoints'].open() as affectedData:
print "%s-%s: %s (%i data points affected)" % (
unicode(e['StartDate'].value), unicode(e['EndDate'].value),
e['Details'].value, len(affectedData))
def LoadNotes(weatherNotes, fileName, weatherData):
with open(fileName, 'r') as f:
id = 1
with weatherNotes.OpenCollection() as collection, weatherData.OpenCollection() as data:
while True:
line = f.readline()
if len(line) == 0:
break
elif line[0] == '#':
continue
noteWords = line.split()
if noteWords:
note = collection.NewEntity()
note['ID'].SetFromValue(id)
start = iso.TimePoint(
date=iso.Date.FromString(noteWords[0]),
time=iso.Time(hour=0, minute=0, second=0))
note['StartDate'].SetFromValue(start)
end = iso.TimePoint(
date=iso.Date.FromString(noteWords[1]).Offset(days=1),
time=iso.Time(hour=0, minute=0, second=0))
note['EndDate'].SetFromValue(end)
note['Details'].SetFromValue(
string.join(noteWords[2:], ' '))
collection.InsertEntity(note)
# now find the data points that match
data.Filter(
core.CommonExpression.FromString(
"TimePoint ge datetime'%s' and TimePoint lt datetime'%s'" %
(unicode(start), unicode(end))))
for dataPoint in data.values():
# use values, not itervalues to avoid this bug in Python 2.7
# http://bugs.python.org/issue10513
dataPoint['Note'].BindEntity(note)
data.UpdateEntity(dataPoint)
id = id + 1
with weatherNotes.OpenCollection() as collection:
collection.OrderBy(
core.CommonExpression.OrderByFromString('StartDate desc'))
for e in collection.itervalues():
with e['DataPoints'].OpenCollection() as affectedData:
print "%s-%s: %s (%i data points affected)" % (
unicode(e['StartDate'].value), unicode(e['EndDate'].value),
e['Details'].value, len(affectedData))
def LogException(self):
now=iso.TimePoint()
now.Now()
e=str(sys.exc_info()[1])
if e:
e="%s::%s"%(str(sys.exc_info()[0]),e)
else:
e=str(sys.exc_info()[0])
self.Log("%s::ERROR::"%str(now)+e)
self.Log(string.join(traceback.format_tb(sys.exc_info()[2]),'\n'))
class MockTime(object):
#: the float time (since the epoch) corresponding to 01 January
#: 1970 00:00:00 UTC the OAuth time origin.
oauth_origin = float(
iso.TimePoint(date=iso.Date(century=19, year=70, month=1, day=1),
time=iso.Time(hour=0, minute=0, second=0,
zdirection=0)).get_unixtime())
def __init__(self, base=None):
if base is None:
self.now = time.time()
else:
self.now = base
def time(self):
return self.now
def tick(self, delta=1.0):
self.now += delta
def LoadDataFromFile(weatherData, f, year, month, day):
with weatherData.OpenCollection() as collection:
while True:
line = f.readline()
if len(line) == 0:
break
elif line[0] == '#':
continue
data = line.split()
if not data:
continue
if len(data) < 11:
data = data + ['*'] * (11 - len(data))
for i in (1, 3, 5, 7, 8, 10):
try:
data[i] = float(data[i])
except ValueError:
data[i] = None
for i in (2, 4, 6):
try:
data[i] = int(data[i])
except ValueError:
data[i] = None
dataPoint = collection.new_entity()
hour, min = map(int, data[0].split(':'))
tValue = iso.TimePoint(date=iso.Date(century=year / 100,
year=year % 100,
month=month,
day=day),
time=iso.Time(hour=hour,
minute=min,
second=0))
bst = IsBST(tValue)
if bst is not False:
# assume BST for now, add the zone info and then shift to GMT
tValue = tValue.WithZone(zDirection=1,
zHour=1).ShiftZone(zDirection=0)
dataPoint['TimePoint'].SetFromValue(tValue)
dataPoint['Temperature'].SetFromValue(data[1])
dataPoint['Humidity'].SetFromValue(data[2])
dataPoint['DewPoint'].SetFromValue(data[3])
dataPoint['Pressure'].SetFromValue(data[4])
dataPoint['WindSpeed'].SetFromValue(data[5])
dataPoint['WindDirection'].SetFromValue(data[6])
dataPoint['Sun'].SetFromValue(data[7])
dataPoint['Rain'].SetFromValue(data[8])
shour, smin = map(int, data[9].split(':'))
dataPoint['SunRainStart'].SetFromValue(
iso.Time(hour=shour, minute=smin, second=0))
dataPoint['WindSpeedMax'].SetFromValue(data[10])
try:
collection.insert_entity(dataPoint)
except edm.ConstraintError:
if bst is None:
# This was an ambiguous entry, the first one is in BST, the
# second one is in GMT as the clocks have gone back, so we
# shift forward again and then force the zone to GMT
tValue = tValue.ShiftZone(zDirection=1,
zHour=1).WithZone(zDirection=0)
dataPoint['TimePoint'].SetFromValue(tValue)
logging.info("Auto-detecting switch to GMT at: %s",
str(tValue))
try:
collection.insert_entity(dataPoint)
except KeyError:
logging.error(
"Duplicate data point during BST/GMT switching: %s",
str(tValue))
else:
logging.error("Unexpected duplicate data point: %s",
str(tValue))
def __init__(self, parent):
AtomElement.__init__(self, parent)
#: a :py:class:`~pyslet.iso8601.TimePoint` instance representing this date
self.date = iso8601.TimePoint()
def load_data_from_file(weather_data, f, year, month, day):
with weather_data.open() as collection:
while True:
line = f.readline().decode('ascii')
if len(line) == 0 or line.startswith('Date unknown.'):
break
elif line[0] == '#':
continue
data = line.split()
if not data:
continue
if len(data) < 11:
data = data + ['*'] * (11 - len(data))
for i in (1, 3, 5, 7, 8, 10):
try:
data[i] = float(data[i])
except ValueError:
data[i] = None
for i in (2, 4):
try:
data[i] = int(data[i])
except ValueError:
data[i] = None
data[6] = data[6].strip()
data_point = collection.new_entity()
hour, min = [int(i) for i in data[0].split(':')]
tvalue = iso.TimePoint(
date=iso.Date(century=year // 100, year=year %
100, month=month, day=day),
time=iso.Time(hour=hour, minute=min, second=0))
bst = is_bst(tvalue)
if bst is not False:
# assume BST for now, add the zone info and then shift to GMT
tvalue = tvalue.with_zone(
zdirection=1, zhour=1).shift_zone(zdirection=0)
data_point['TimePoint'].set_from_value(tvalue)
data_point['Temperature'].set_from_value(data[1])
data_point['Humidity'].set_from_value(data[2])
data_point['DewPoint'].set_from_value(data[3])
data_point['Pressure'].set_from_value(data[4])
data_point['WindSpeed'].set_from_value(data[5])
data_point['WindDirection'].set_from_value(data[6])
data_point['Sun'].set_from_value(data[7])
data_point['Rain'].set_from_value(data[8])
shour, smin = [int(i) for i in data[9].split(':')]
data_point['SunRainStart'].set_from_value(
iso.Time(hour=shour, minute=smin, second=0))
data_point['WindSpeedMax'].set_from_value(data[10])
try:
collection.insert_entity(data_point)
except edm.ConstraintError:
if bst is None:
# This was an ambiguous entry, the first one is in
# BST, the second one is in GMT as the clocks have
# gone back, so we shift forward again and then
# force the zone to GMT
tvalue = tvalue.shift_zone(
zdirection=1, zhour=1).with_zone(zdirection=0)
data_point['TimePoint'].set_from_value(tvalue)
logging.info(
"Auto-detecting switch to GMT at: %s", str(tvalue))
try:
collection.insert_entity(data_point)
except KeyError:
logging.error("Duplicate data point during BST/GMT "
"switching: %s", str(tvalue))
else:
logging.error(
"Unexpected duplicate data point: %s", str(tvalue))