def extractRecordFrom(self, archive, time_ts):
"""Get a record from the archive database.
This is a general version that works for:
- WeatherUnderground
- PWSweather
- CWOP
It can be overridden and specialized for additional protocols.
archive: An instance of weewx.archive.Archive
time_ts: The record desired as a unix epoch time.
returns: A dictionary of weather values"""
sod_ts = weeutil.weeutil.startOfDay(time_ts)
# Get the data record off the archive database:
sqlrec = archive.getSql(REST.sql_select, (time_ts,))
# There is no reason why the record would not be in the database,
# but check just in case:
if sqlrec is None:
raise SkippedPost("Non existent record %s" % (weeutil.weeutil.timestamp_to_string(time_ts),))
# Make a dictionary out of the types:
datadict = dict(zip(REST.archive_types, sqlrec))
# CWOP says rain should be "rain that fell in the past hour". WU says
# it should be "the accumulated rainfall in the past 60 min".
# Presumably, this is exclusive of the archive record 60 minutes before,
# so the SQL statement is exclusive on the left, inclusive on the right.
datadict['rain'] = archive.getSql("SELECT SUM(rain) FROM archive WHERE dateTime>? AND dateTime<=?",
(time_ts - 3600.0, time_ts))[0]
# Similar issue, except for last 24 hours:
datadict['rain24'] = archive.getSql("SELECT SUM(rain) FROM archive WHERE dateTime>? AND dateTime<=?",
(time_ts - 24*3600.0, time_ts))[0]
# NB: The WU considers the archive with time stamp 00:00 (midnight) as
# (wrongly) belonging to the current day (instead of the previous
# day). But, it's their site, so we'll do it their way. That means the
# SELECT statement is inclusive on both time ends:
datadict['dayRain'] = archive.getSql("SELECT SUM(rain) FROM archive WHERE dateTime>=? AND dateTime<=?",
(sod_ts, time_ts))[0]
# All these online weather sites require US units.
if datadict['usUnits'] == weewx.US:
# It's already in US units.
return datadict
else:
# It's in something else. Perform the conversion
datadict_us = weewx.units.StdUnitConverters[weewx.US].convertDict(datadict)
# Add the new unit system
datadict_us['usUnits'] = weewx.US
return datadict_us
def test_get_records(self):
# Add a bunch of records:
with weewx.archive.Archive.open_with_create(self.archive_db_dict, archive_schema) as archive:
archive.addRecord(genRecords())
# Now fetch them:
with weewx.archive.Archive.open(self.archive_db_dict) as archive:
# Test getSql:
bar0 = archive.getSql("SELECT barometer FROM archive WHERE dateTime=?", (start_ts,))
self.assertEqual(bar0[0], barfunc(0))
# Test genSql:
for (irec,_row) in enumerate(archive.genSql("SELECT barometer FROM archive;")):
self.assertEqual(_row[0], barfunc(irec))
# Try getRecord():
target_ts = timevec[nrecs/2]
_rec = archive.getRecord(target_ts)
# Check that the missing windSpeed is None, then remove it in order to do the compare:
self.assertEqual(_rec.pop('windSpeed'), None)
self.assertEqual(expected_record(nrecs/2), _rec)
# Try finding the nearest neighbor below
target_ts = timevec[nrecs/2] + interval/100
_rec = archive.getRecord(target_ts, max_delta=interval/50)
# Check that the missing windSpeed is None, then remove it in order to do the compare:
self.assertEqual(_rec.pop('windSpeed'), None)
self.assertEqual(expected_record(nrecs/2), _rec)
# Try finding the nearest neighbor above
target_ts = timevec[nrecs/2] - interval/100
_rec = archive.getRecord(target_ts, max_delta=interval/50)
# Check that the missing windSpeed is None, then remove it in order to do the compare:
self.assertEqual(_rec.pop('windSpeed'), None)
self.assertEqual(expected_record(nrecs/2), _rec)
# Try finding a neighbor too far away:
target_ts = timevec[nrecs/2] - interval/2
_rec = archive.getRecord(target_ts, max_delta=interval/50)
self.assertEqual(_rec, None)
# Now try fetching them as vectors:
with weewx.archive.Archive.open(self.archive_db_dict) as archive:
barvec = archive.getSqlVectors('barometer', start_ts, stop_ts)
# Recall that barvec will be a 2-way tuple. The first element is the value tuple of
# the time vector, the second of the data vector.
self.assertEqual(barvec[0], ([timefunc(irec) for irec in range(nrecs)], "unix_epoch", "group_time"))
self.assertEqual(barvec[1], ([barfunc(irec) for irec in range(nrecs)], "inHg", "group_pressure"))
# Now try fetching the vectora gain, but using aggregation.
# Start by setting up a generator function that will return the records to be
# included in each aggregation
gen = gen_included_recs(timevec, start_ts, stop_ts, 6*interval)
with weewx.archive.Archive.open(self.archive_db_dict) as archive:
barvec = archive.getSqlVectors('barometer', start_ts, stop_ts, aggregate_interval=6*interval, aggregate_type='avg')
n_expected = int(nrecs / 6)
self.assertEqual(n_expected, len(barvec[0][0]))
for irec in range(n_expected):
# Get the set of records to be included in this aggregation:
recs = gen.next()
# Make sure the timestamp of the aggregation interval is the same as the last
# record to be included in the aggregation:
self.assertEqual(timevec[max(recs)], barvec[0][0][irec])
# Calculate the expected average of the records included in the aggregation.
expected_avg = sum((barfunc(i) for i in recs)) / len(recs)
# Compare them.
self.assertAlmostEqual(expected_avg, barvec[1][0][irec])
def test(config_path):
weewx.debug = 1
try :
config_dict = configobj.ConfigObj(config_path, file_error=True)
except IOError:
print "Unable to open configuration file ", config_path
exit()
# Open up the main archive database
archiveFilename = os.path.join(config_dict['Station']['WEEWX_ROOT'],
config_dict['Archive']['archive_file'])
archive = weewx.archive.Archive(archiveFilename)
statsFilename = os.path.join(config_dict['Station']['WEEWX_ROOT'],
config_dict['Stats']['stats_file'])
conn = sqlite3.connect(statsFilename)
conn.row_factory = sqlite3.Row
cursor=conn.cursor()
# Start out by getting a date that exists in the database.
# Select the 10th record to check.
cursor.execute("SELECT dateTime FROM barometer LIMIT 2 OFFSET 10")
# This gets the start time of the test day:
testrow = cursor.fetchone()
if not testrow:
print "Cannot get a test row from the typeStats database. Perhaps it is not initialized??"
exit()
start_ts = testrow['dateTime']
# And this gets the end of the test day:
testrow = cursor.fetchone()
stop_ts = testrow['dateTime']
# No need to leave these lying around:
cursor.close()
conn.close()
print "start time=", weeutil.weeutil.timestamp_to_string(start_ts)
print "stop time= ", weeutil.weeutil.timestamp_to_string(stop_ts)
# Make sure it's a start of day:
assert(start_ts == weeutil.weeutil.startOfDay(start_ts))
# OK, now open up the typeStats database using the class StatsReadonlyDb:
statsDb = StatsReadonlyDb(statsFilename)
allStats = statsDb.day(start_ts)
# Test it against some types
# Should really do a test for 'wind' as well.
# Should also test monthly, yearly summaries
for stats_type in ('barometer', 'outTemp', 'inTemp', 'heatindex'):
print "\n***************** %s ********************" % stats_type
# Get the StatsDict for this day and this stats_type:
typeStats = statsDb.getStatsForType(stats_type, start_ts)
# Now test all the aggregates:
for aggregate in ('min', 'max', 'sum', 'count'):
# Compare to the main archive:
res = archive.getSql("SELECT %s(%s) FROM archive WHERE dateTime>? AND dateTime <=?;" % (aggregate, stats_type), start_ts, stop_ts)
# From StatsDb:
typeStats_res = typeStats.__getattribute__(aggregate)
allStats_res = allStats[stats_type].__getattribute__(aggregate)
print "%s: results from stats database using getStatsForType(): " % aggregate, typeStats_res
print "%s: results from stats database using day(): " % aggregate, allStats_res
print "%s: result from running SQL on the main archive: " % aggregate, res[0]
assert(typeStats_res == res[0])
assert(allStats_res == res[0])
# Check the times of min and max as well:
if aggregate in ('min','max'):
res2 = archive.getSql("SELECT dateTime FROM archive WHERE %s = ? AND dateTime>? AND dateTime <=?" % (stats_type,), res[0], start_ts, stop_ts)
stats_time = typeStats.__getattribute__(aggregate+'time')
print aggregate+'time: from main archive: ', weeutil.weeutil.timestamp_to_string(res2[0])
print aggregate+'time: from typeStats database: ', weeutil.weeutil.timestamp_to_string(stats_time)
assert( stats_time == res2[0])
print "PASSES\n"