def gen_data(connect): """Function for generating fake data and putting it in the database in 6NF.""" t0 = time.time() cursor = connect.cursor() connect.begin() N = 0 for packet in genFakeRecords(start_ts=start_ts, stop_ts=stop_ts, interval=interval): # Break the packet into separate observation type for obstype in ['outTemp', 'barometer', 'windSpeed', 'windDir', 'windGust', 'windGustDir', 'rain']: # Put each observation type in its own row cursor.execute("INSERT INTO bench VALUES (?, ?, ?);", (packet['dateTime'], obstype, packet[obstype])) N += 1 # Commit every 32,0000 inserts. if (N % 32000) == 0 : connect.commit() connect.begin() print >> sys.stdout, "packets processed: %d; Last date: %s\r" % \ (N, weeutil.weeutil.timestamp_to_string(packet["dateTime"])), sys.stdout.flush() connect.commit() cursor.close() t1 = time.time() print "\n%d records generated in %.1f seconds" % (N, t1-t0)
def gen_data(client): """Function for generating fake data and putting it in the influxdb database.""" # Create a giant JSON structure, then commit in one write. Much faster. giant_json = [] N = 0 for packet in genFakeRecords(start_ts=start_ts, stop_ts=stop_ts, interval=interval): print >> sys.stdout, "packets processed: %d; Last date: %s\r" % \ (N, weeutil.weeutil.timestamp_to_string(packet["dateTime"])), sys.stdout.flush() json_body = { "measurement": "wxpacket", "tags": { "instrumentID": 1, }, "time": packet["dateTime"] * 1000000000, # Convert to nanoseconds "fields": { "interval" : packet["interval"], "usUnits" : packet["usUnits"], "outTemp" : packet["outTemp"], "barometer" : packet["barometer"], "windSpeed" : packet["windSpeed"], "windDir" : packet["windDir"], "windGust" : packet["windSpeed"], "windGustDir" : packet["windDir"], "rain" : packet["rain"] } } giant_json.append(json_body) # Do the write of the whole structure in a single commit: client.write_points(giant_json)
def setUp(self): # The data set is a list of faked records at 5 second intervals. The stage of the weather cycle # is set so that some rain will appear. self.dataset = list( genFakeRecords(start_ts=start_ts + 5, stop_ts=stop_ts, interval=5, weather_phase_offset=gen_fake_data.weather_cycle * math.pi / 2.0))
def setup_database(db_dict): """Set up a database by using addRecord()""" try: # Drop the old database weedb.drop(db_dict) except weedb.NoDatabaseError: pass # Get a new database by initializing with the schema db_manager = weewx.manager.DaySummaryManager.open_with_create(db_dict, schema=schema) # Populate the database. By passing in a generator, it is all done as one transaction. db_manager.addRecord(gen_fake_data.genFakeRecords(start_ts, stop_ts, interval=interval_secs)) return db_manager
def setUp(self): """Set up an in-memory database""" self.db_manager = weewx.manager.Manager.open_with_create( { 'database_name': ':memory:', 'driver': 'weedb.sqlite' }, schema=schemas.wview_extended.schema) # Populate the database with 60 minutes worth of data at 5 minute intervals. Set the annual # phase to half a year, so that the temperatures will be high for record in gen_fake_data.genFakeRecords(TestET.start, TestET.start + 3600, interval=300, annual_phase_offset=math.pi * ( 24.0 * 3600 * 365)): # Add some radiation and humidity: record['radiation'] = 860 record['outHumidity'] = 50 self.db_manager.addRecord(record)
def gen_data(collection): """Function for generating fake data and putting it in the MongoDB database.""" # Drop any existing collection, then rebuild it collection.drop() t0 = time.time() N = 0 for record in genFakeRecords(start_ts=start_ts, stop_ts=stop_ts, interval=interval): if not N % 100: print >> sys.stdout, "packets processed: %d; Last date: %s\r" % \ (N, weeutil.weeutil.timestamp_to_string(record["dateTime"])), sys.stdout.flush() record['dateTime'] = datetime.datetime.utcfromtimestamp(record['dateTime']) collection.insert_one(record) N += 1 # Build an index on the timestamp collection.create_index("dateTime", unique=True) t1 = time.time() print "Finished creating collection. Elapsed time ", (t1 - t0)
def setUp(self): self.dataset = [record for record in genFakeRecords(start_ts=start_ts, stop_ts=stop_ts, interval=5)]
def setUp(self): # The data set is a list of faked records at 5 second intervals self.dataset = list( genFakeRecords(start_ts=start_ts + 5, stop_ts=stop_ts, interval=5))
def setUp(self): self.dataset = [ record for record in genFakeRecords( start_ts=start_ts, stop_ts=stop_ts, interval=5) ]