def gen_data(connect):
"""Function for generating fake data and putting it in the database in 6NF."""
t0 = time.time()
cursor = connect.cursor()
connect.begin()
N = 0
for packet in genFakeRecords(start_ts=start_ts, stop_ts=stop_ts, interval=interval):
# Break the packet into separate observation type
for obstype in ['outTemp', 'barometer', 'windSpeed', 'windDir', 'windGust', 'windGustDir', 'rain']:
# Put each observation type in its own row
cursor.execute("INSERT INTO bench VALUES (?, ?, ?);", (packet['dateTime'], obstype, packet[obstype]))
N += 1
# Commit every 32,0000 inserts.
if (N % 32000) == 0 :
connect.commit()
connect.begin()
print >> sys.stdout, "packets processed: %d; Last date: %s\r" % \
(N, weeutil.weeutil.timestamp_to_string(packet["dateTime"])),
sys.stdout.flush()
connect.commit()
cursor.close()
t1 = time.time()
print "\n%d records generated in %.1f seconds" % (N, t1-t0)
def gen_data(client):
"""Function for generating fake data and putting it in the influxdb database."""
# Create a giant JSON structure, then commit in one write. Much faster.
giant_json = []
N = 0
for packet in genFakeRecords(start_ts=start_ts, stop_ts=stop_ts, interval=interval):
print >> sys.stdout, "packets processed: %d; Last date: %s\r" % \
(N, weeutil.weeutil.timestamp_to_string(packet["dateTime"])),
sys.stdout.flush()
json_body = {
"measurement": "wxpacket",
"tags": {
"instrumentID": 1,
},
"time": packet["dateTime"] * 1000000000, # Convert to nanoseconds
"fields": {
"interval" : packet["interval"],
"usUnits" : packet["usUnits"],
"outTemp" : packet["outTemp"],
"barometer" : packet["barometer"],
"windSpeed" : packet["windSpeed"],
"windDir" : packet["windDir"],
"windGust" : packet["windSpeed"],
"windGustDir" : packet["windDir"],
"rain" : packet["rain"]
}
}
giant_json.append(json_body)
# Do the write of the whole structure in a single commit:
client.write_points(giant_json)
def setUp(self):
# The data set is a list of faked records at 5 second intervals. The stage of the weather cycle
# is set so that some rain will appear.
self.dataset = list(
genFakeRecords(start_ts=start_ts + 5,
stop_ts=stop_ts,
interval=5,
weather_phase_offset=gen_fake_data.weather_cycle *
math.pi / 2.0))
def setup_database(db_dict):
"""Set up a database by using addRecord()"""
try:
# Drop the old database
weedb.drop(db_dict)
except weedb.NoDatabaseError:
pass
# Get a new database by initializing with the schema
db_manager = weewx.manager.DaySummaryManager.open_with_create(db_dict, schema=schema)
# Populate the database. By passing in a generator, it is all done as one transaction.
db_manager.addRecord(gen_fake_data.genFakeRecords(start_ts, stop_ts, interval=interval_secs))
return db_manager
def setUp(self):
"""Set up an in-memory database"""
self.db_manager = weewx.manager.Manager.open_with_create(
{
'database_name': ':memory:',
'driver': 'weedb.sqlite'
},
schema=schemas.wview_extended.schema)
# Populate the database with 60 minutes worth of data at 5 minute intervals. Set the annual
# phase to half a year, so that the temperatures will be high
for record in gen_fake_data.genFakeRecords(TestET.start, TestET.start + 3600, interval=300,
annual_phase_offset=math.pi * (
24.0 * 3600 * 365)):
# Add some radiation and humidity:
record['radiation'] = 860
record['outHumidity'] = 50
self.db_manager.addRecord(record)
def gen_data(collection):
"""Function for generating fake data and putting it in the MongoDB database."""
# Drop any existing collection, then rebuild it
collection.drop()
t0 = time.time()
N = 0
for record in genFakeRecords(start_ts=start_ts, stop_ts=stop_ts, interval=interval):
if not N % 100:
print >> sys.stdout, "packets processed: %d; Last date: %s\r" % \
(N, weeutil.weeutil.timestamp_to_string(record["dateTime"])),
sys.stdout.flush()
record['dateTime'] = datetime.datetime.utcfromtimestamp(record['dateTime'])
collection.insert_one(record)
N += 1
# Build an index on the timestamp
collection.create_index("dateTime", unique=True)
t1 = time.time()
print "Finished creating collection. Elapsed time ", (t1 - t0)
def setUp(self):
self.dataset = [record for record in genFakeRecords(start_ts=start_ts, stop_ts=stop_ts, interval=5)]
def setUp(self):
# The data set is a list of faked records at 5 second intervals
self.dataset = list(
genFakeRecords(start_ts=start_ts + 5, stop_ts=stop_ts, interval=5))
def setUp(self):
self.dataset = [
record for record in genFakeRecords(
start_ts=start_ts, stop_ts=stop_ts, interval=5)
]
