def test_add_archive_records(self):
# Test adding records using a 'with' statement:
with weewx.archive.Archive.open_with_create(self.archive_db_dict, archive_schema) as archive:
archive.addRecord(genRecords())
# Now test to see what's in there:
with weewx.archive.Archive.open(self.archive_db_dict) as archive:
self.assertEqual(archive.firstGoodStamp(), start_ts)
self.assertEqual(archive.lastGoodStamp(), stop_ts)
self.assertEqual(archive.std_unit_system, std_unit_system)
expected_iterator = genRecords()
for _rec in archive.genBatchRecords():
try:
_expected_rec = expected_iterator.next()
except StopIteration:
break
# Check that the missing windSpeed is None, then remove it in order to do the compare:
self.assertEqual(_rec.pop('windSpeed'), None)
self.assertEqual(_expected_rec, _rec)
# Test adding an existing record. It should just quietly swallow it:
existing_record = {'dateTime': start_ts, 'interval': interval, 'usUnits' : 1, 'outTemp': 68.0}
archive.addRecord(existing_record)
# Test changing the unit system. It should raise a ValueError exception:
metric_record = {'dateTime': stop_ts + interval, 'interval': interval, 'usUnits' : 16, 'outTemp': 20.0}
self.assertRaises(ValueError, archive.addRecord, metric_record)
def loader(config_dict, engine):
# This loader uses a bit of a hack to have the simulator resume at a later
# time. It's not bad, but I'm not enthusiastic about having special
# knowledge about the database in a driver, albeit just the loader.
start_ts = resume_ts = None
if 'start' in config_dict['Simulator']:
# A start has been specified. Extract the time stamp.
start_tt = time.strptime(config_dict['Simulator']['start'], "%Y-%m-%d %H:%M")
start_ts = time.mktime(start_tt)
# If the 'resume' keyword is present and True, then get the last archive
# record out of the database and resume with that.
if int(config_dict['Simulator'].get('resume', 1)):
import weewx.archive
# Resume with the last time in the database. If there is no such
# time, then fall back to the time specified in the configuration
# dictionary.
archive_db = config_dict['StdArchive']['archive_database']
archive_db_dict = config_dict['Databases'][archive_db]
try:
with weewx.archive.Archive.open(archive_db_dict) as archive:
resume_ts = archive.lastGoodStamp()
except weedb.OperationalError:
pass
else:
# The resume keyword is not present. Start with the seed time:
resume_ts = start_ts
station = Simulator(start_time=start_ts, resume_time=resume_ts, **config_dict['Simulator'])
return station
def postData(self, archive, time_ts):
"""Post data to CWOP, using the CWOP protocol."""
_last_ts = archive.lastGoodStamp()
# There are a variety of reasons to skip a post to CWOP.
# 1. They do not allow backfilling, so there is no reason
# to post anything other than the latest record:
if time_ts != _last_ts:
raise SkippedPost, "CWOP: Record %s is not last record" %\
(weeutil.weeutil.timestamp_to_string(time_ts), )
# 2. No reason to post an old out-of-date record.
_how_old = time.time() - time_ts
if self.stale and _how_old > self.stale:
raise SkippedPost, "CWOP: Record %s is stale (%d > %d)." %\
(weeutil.weeutil.timestamp_to_string(time_ts), _how_old, self.stale)
# 3. Finally, we don't want to post more often than the interval
if self._lastpost and time_ts - self._lastpost < self.interval:
raise SkippedPost, "CWOP: Wait interval (%d) has not passed." %\
(self.interval, )
# Get the data record for this time:
_record = self.extractRecordFrom(archive, time_ts)
# Send it to its destination:
self.sendRecord(_record)
self._lastpost = time_ts
def loader(config_dict, engine):
# This loader uses a bit of a hack to have the simulator resume at a later
# time. It's not bad, but I'm not enthusiastic about having special
# knowledge about the database in a driver, albeit just the loader.
start_ts = resume_ts = None
if 'start' in config_dict['Simulator']:
# A start has been specified. Extract the time stamp.
start_tt = time.strptime(config_dict['Simulator']['start'], "%Y-%m-%d %H:%M")
start_ts = time.mktime(start_tt)
# If the 'resume' keyword is present and True, then get the last archive
# record out of the database and resume with that.
if int(config_dict['Simulator'].get('resume', 1)):
import weewx.archive
# Resume with the last time in the database. If there is no such
# time, then fall back to the time specified in the configuration
# dictionary.
archive_db = config_dict['StdArchive']['archive_database']
archive_db_dict = config_dict['Databases'][archive_db]
try:
with weewx.archive.Archive.open(archive_db_dict) as archive:
resume_ts = archive.lastGoodStamp()
except weedb.OperationalError:
pass
else:
# The resume keyword is not present. Start with the seed time:
resume_ts = start_ts
station = Simulator(start_time=start_ts, resume_time=resume_ts, **config_dict['Simulator'])
return station
def main():
usage_string ="""Usage:
restful.py config_path upload-site [--today] [--last]
Arguments:
config_path: Path to weewx.conf
upload-site: Either "Wunderground", "PWSweather", or "CWOP"
Options:
--today: Publish all of today's day
--last: Just do the last archive record. [default]
"""
parser = OptionParser(usage=usage_string)
parser.add_option("-t", "--today", action="store_true", dest="do_today", help="Publish today\'s records")
parser.add_option("-l", "--last", action="store_true", dest="do_last", help="Publish the last archive record only")
(options, args) = parser.parse_args()
if len(args) < 2:
sys.stderr.write("Missing argument(s).\n")
sys.stderr.write(parser.parse_args(["--help"]))
exit()
if options.do_today and options.do_last:
sys.stderr.write("Choose --today or --last, not both\n")
sys.stderr.write(parser.parse_args(["--help"]))
exit()
if not options.do_today and not options.do_last:
options.do_last = True
config_path = args[0]
site = args[1]
weewx.debug = 1
try :
config_dict = configobj.ConfigObj(config_path, file_error=True)
except IOError:
print "Unable to open configuration file ", config_path
exit()
# Open up the main database archive
archiveFilename = os.path.join(config_dict['Station']['WEEWX_ROOT'],
config_dict['Archive']['archive_file'])
archive = weewx.archive.Archive(archiveFilename)
stop_ts = archive.lastGoodStamp()
start_ts = weeutil.weeutil.startOfDay(stop_ts) if options.do_today else stop_ts
publish(config_dict, site, archive, start_ts, stop_ts )
def run(self):
self.setup()
# Open up the main database archive
archiveFilename = os.path.join(
self.config_dict["Station"]["WEEWX_ROOT"], self.config_dict["Archive"]["archive_file"]
)
archive = weewx.archive.Archive(archiveFilename)
stop_ts = archive.lastGoodStamp() if self.gen_ts is None else self.gen_ts
# Generate any images
self.genImages(archive, stop_ts)
def getCurrentRec(self):
# Open up the main database archive
archiveFilename = os.path.join(self.config_dict['Station']['WEEWX_ROOT'],
self.config_dict['Archive']['archive_file'])
archive = weewx.archive.Archive(archiveFilename)
self.stop_ts = archive.lastGoodStamp() if self.gen_ts is None else self.gen_ts
self.start_ts = archive.firstGoodStamp()
# Get a dictionary with the current record:
current_dict = archive.getRecord(self.stop_ts)
# Wrap it in a ValueDict
currentRec = weewx.units.ValueDict(current_dict, self.unit_info)
return currentRec
def configDatabases(archive_db_dict, stats_db_dict):
"""Configures the main and stats databases."""
# Check to see if it already exists and is configured correctly.
try:
with weewx.archive.Archive.open(archive_db_dict) as archive:
if archive.firstGoodStamp() == start_ts and archive.lastGoodStamp() == stop_ts:
# Database already exists. We're done.
return
except:
pass
# Delete anything that might already be there.
try:
weedb.drop(archive_db_dict)
except:
pass
# Now build a new one:
with weewx.archive.Archive.open_with_create(archive_db_dict, user.schemas.defaultArchiveSchema) as archive:
# Because this can generate voluminous log information,
# suppress all but the essentials:
syslog.setlogmask(syslog.LOG_UPTO(syslog.LOG_ERR))
# Now generate and add the fake records to populate the database:
t1= time.time()
archive.addRecord(genFakeRecords())
t2 = time.time()
print "Time to create synthetic archive database = %6.2fs" % (t2-t1,)
# Now go back to regular logging:
syslog.setlogmask(syslog.LOG_UPTO(syslog.LOG_DEBUG))
# Delete any old stats database:
try:
weedb.drop(stats_db_dict)
except weedb.NoDatabase:
pass
# Now create and configure a new one:
with weewx.stats.StatsDb.open_with_create(stats_db_dict, user.schemas.defaultStatsSchema) as stats:
t1 = time.time()
# Now backfill the stats database from the main archive database.
nrecs = stats.backfillFrom(archive)
t2 = time.time()
print "Time to backfill stats database with %d records: %6.2fs" % (nrecs, t2-t1)
def test_empty_archive(self):
archive = weewx.archive.Archive.open_with_create(self.archive_db_dict, archive_schema)
self.assertEqual(archive.firstGoodStamp(), None)
self.assertEqual(archive.lastGoodStamp(), None)
self.assertEqual(archive.getRecord(123456789), None)
self.assertEqual(archive.getRecord(123456789, max_delta=1800), None)
# python /home/weewx/bin/examples/thisfile.py /home/weewx/weewx.conf
#
import string
import weeutil.weeutil
import weewx.stats
import sys
if __name__ == '__main__':
# get the min/max timestamp from the archive
import weewx.archive
archive = weewx.archive.Archive('/home/weewx/archive/weewx.sdb')
myMinStamp = archive.firstGoodStamp()
myMaxStamp = archive.lastGoodStamp()
timeSpan = weeutil.weeutil.TimeSpan(myMinStamp, myMaxStamp)
statsdb = weewx.stats.StatsReadonlyDb('/tmp/stats.sdb')
spanStats = weewx.stats.TimeSpanStats(statsdb, timeSpan)
# Print maximum temperature for each month in the year:
print "========================="
print "outTemp max/min per month"
print "========================="
for monthStats in spanStats.months:
m = monthStats.outTemp.maxtime.format("%Y-%m")
print "%s : %s %s" % (m, monthStats.outTemp.max,
monthStats.outTemp.min)
def genImages(self, archive, time_ts):
"""Generate the images.
The time scales will be chosen to include the given timestamp, with nice beginning
and ending times.
time_ts: The time around which plots are to be generated. This will also be used as
the bottom label in the plots. [optional. Default is to use the time of the last record
in the archive database.]
"""
t1 = time.time()
ngen = 0
if not time_ts:
time_ts = archive.lastGoodStamp()
if not time_ts:
time_ts = time.time()
# Loop over each time span class (day, week, month, etc.):
for timespan in self.image_dict.sections:
# Now, loop over all plot names in this time span class:
for plotname in self.image_dict[timespan].sections:
# Accumulate all options from parent nodes:
plot_options = weeutil.weeutil.accumulateLeaves(self.image_dict[timespan][plotname])
image_root = os.path.join(self.weewx_root, plot_options["HTML_ROOT"])
# Get the path of the file that the image is going to be saved to:
img_file = os.path.join(image_root, "%s.png" % plotname)
# Check whether this plot needs to be done at all:
ai = plot_options.as_int("aggregate_interval") if plot_options.has_key("aggregate_interval") else None
if skipThisPlot(time_ts, ai, img_file):
continue
# Create the subdirectory that the image is to be put in.
# Wrap in a try block in case it already does.
try:
os.makedirs(os.path.dirname(img_file))
except:
pass
# Calculate a suitable min, max time for the requested time span
(minstamp, maxstamp, timeinc) = weeplot.utilities.scaletime(
time_ts - plot_options.as_int("time_length"), time_ts
)
# Create a new instance of a time plot and start adding to it
plot = weeplot.genplot.TimePlot(plot_options)
# Set the min, max time axis
plot.setXScaling((minstamp, maxstamp, timeinc))
# Set the y-scaling, using any user-supplied hints:
plot.setYScaling(weeutil.weeutil.convertToFloat(plot_options.get("yscale")))
# Get a suitable bottom label:
bottom_label_format = plot_options.get("bottom_label_format", "%m/%d/%y %H:%M")
bottom_label = time.strftime(bottom_label_format, time.localtime(time_ts))
plot.setBottomLabel(bottom_label)
# Loop over each line to be added to the plot.
for line_name in self.image_dict[timespan][plotname].sections:
# Accumulate options from parent nodes.
line_options = weeutil.weeutil.accumulateLeaves(self.image_dict[timespan][plotname][line_name])
# See what SQL variable type to use for this line. By default,
# use the section name.
var_type = line_options.get("data_type", line_name)
# Add a unit label. NB: all will get overwritten except the last.
# Get the label from the configuration dictionary.
# TODO: Allow multiple unit labels, one for each plot line?
unit_label = line_options.get("y_label", self.unit_label_dict.get(var_type, ""))
# PIL cannot handle UTF-8. So, convert to Latin1. Also, strip off
# any leading and trailing whitespace so it's easy to center
unit_label = weeutil.weeutil.utf8_to_latin1(unit_label).strip()
plot.setUnitLabel(unit_label)
# See if a line label has been explicitly requested:
label = line_options.get("label")
if not label:
# No explicit label. Is there a generic one?
# If not, then the SQL type will be used instead
label = self.title_dict.get(var_type, var_type)
# Convert to Latin-1
label = weeutil.weeutil.utf8_to_latin1(label)
# See if a color has been explicitly requested.
color_str = line_options.get("color")
color = int(color_str, 0) if color_str is not None else None
# Get the line width, if explicitly requested.
width_str = line_options.get("width")
width = int(width_str) if width_str is not None else None
# Get the type of line ("bar', 'line', or 'vector')
line_type = line_options.get("plot_type", "line")
if line_type == "vector":
vector_rotate_str = line_options.get("vector_rotate")
vector_rotate = -float(vector_rotate_str) if vector_rotate_str is not None else None
else:
vector_rotate = None
# Look for aggregation type:
aggregate_type = line_options.get("aggregate_type")
if aggregate_type in (None, "", "None", "none"):
# No aggregation specified.
aggregate_type = None
aggregate_interval = None
else:
try:
# Aggregation specified. Get the interval.
aggregate_interval = line_options.as_int("aggregate_interval")
except KeyError:
syslog.syslog(
syslog.LOG_ERR,
"genimages: aggregate interval required for aggregate type %s" % aggregate_type,
)
syslog.syslog(syslog.LOG_ERR, "genimages: line type %s skipped" % var_type)
continue
# Get the time and data vectors from the database:
(time_vec_t, data_vec_t) = archive.getSqlVectorsExtended(
var_type, minstamp, maxstamp, aggregate_interval, aggregate_type
)
new_time_vec_t = self.unit_info.convert(time_vec_t)
new_data_vec_t = self.unit_info.convert(data_vec_t)
# Add the line to the emerging plot:
plot.addLine(
weeplot.genplot.PlotLine(
new_time_vec_t[0],
new_data_vec_t[0],
label=label,
color=color,
width=width,
line_type=line_type,
interval=aggregate_interval,
vector_rotate=vector_rotate,
)
)
# OK, the plot is ready. Render it onto an image
image = plot.render()
# Now save the image
image.save(img_file)
ngen += 1
t2 = time.time()
syslog.syslog(syslog.LOG_INFO, "genimages: Generated %d images in %.2f seconds" % (ngen, t2 - t1))
