def test_read(self): # Create records using synthetic, randomized data id and write to db test_num = random.randint(0,100000) parser = NMEAParser() writer = DatabaseWriter(database='test', host='localhost', user='******', password='******') writer.db.exec_sql_command('truncate table data') reader = DatabaseReader(database='test', host='localhost', user='******', password='******') # Write to database, automatically creating table records = [parser.parse_record(s) for s in SAMPLE_DATA] index = 0 for record in records: logging.debug('Writing record "%s"', str(record)) writer.write(record) result = True while result: result = writer.db.read() logging.info('Read %d: %s', index, result) if result: self.assertEqual(result, SAMPLE_RESULTS[index]) index += 1 # Test range: read a range that should include 3 records results = reader.read_range(start=2, stop=5) self.assertEqual(results, {'S330GPSDay': [(1509778447.17313, 7)], 'S330GPSMonth': [(1509778447.17313, 8)], 'S330GPSYear': [(1509778447.17313, 2014)]}) # Next record should be one after that result = reader.read() self.assertEqual(result, {'S330GPSTime': [(1509778449.210395, 2034.17)]}) # Test time_range: read a range that should include 3 records results = reader.read_time_range(start_time=1509778449.210395, stop_time=1509778453.290818) self.assertEqual(results, {'S330GPSTime': [(1509778451.248784, 2035.17), (1509778453.290817, 2036.17)], 'S330GPSDay': [(1509778451.248784, 7), (1509778453.290817, 7)], 'S330GPSMonth': [(1509778451.248784, 8), (1509778453.290817, 8)], 'S330GPSYear': [(1509778451.248784, 2014), (1509778453.290817, 2014)]}) # Next record should be one after that result = reader.read() self.assertEqual(result, {'S330GPSTime': [(1509778455.328116, 2037.17)]}) writer.db.close() reader.db.close()
else: readers.append(CachedDataReader(subscription=subscription)) # For each comma-separated spec, parse out values for # user@host:database:data_id[:message_type]. We count on # --database_password having been specified somewhere. if new_args.database: password = all_args.database_password (user, host_db) = new_args.database.split('@') (host, database) = host_db.split(':', maxsplit=1) if ':' in database: (database, fields) = database.split(':') else: fields = None readers.append(DatabaseReader(fields=fields, database=database, host=host, user=user, password=password)) # SerialReader is a little more complicated than other readers # because it can take so many parameters. Use the kwargs trick to # pass them all in. if new_args.serial: kwargs = {} for pair in new_args.serial.split(','): (key, value) = pair.split('=') kwargs[key] = value readers.append(SerialReader(**kwargs)) ########################## # Transforms if new_args.slice:
async def serve_fields(self, field_list): """Serve data, if it exists, from database, if it exists, using default database location, tables, user and password. NOTE: This is the kind of code your mother warned you about. It's a quick first pass, and will therefore follow me to my grave and haunt you for years to come. For the love of Guido, please clean this up. """ for (field_name, num_secs) in field_list: logging.info('Requesting field: %s, %g secs.', field_name, num_secs) # Get requested back data. Note that we may have had different # back data time spans for different fields. Because some of these # might be extremely voluminous (think 30 minutes of winch data), # take the computational hit of initially creating a separate # reader for each backlog. fields = [] back_data = {} for (field_name, num_secs) in field_list: fields.append(field_name) if not num_secs in back_data: back_data[num_secs] = [] back_data[num_secs].append(field_name) results = {} now = time.time() for (num_secs, field_list) in back_data.items(): # Create a DatabaseReader to get num_secs worth of back data for # these fields. Provide a start_time of num_secs ago, and no # stop_time, so we get everything up to present. logging.debug('Creating DatabaseReader for %s', field_list) logging.debug('Requesting %g seconds of timestamps from %f-%f', num_secs, now - num_secs, now) reader = DatabaseReader(fields, self.database, self.host, self.user, self.password) num_sec_results = reader.read_time_range(start_time=now - num_secs) logging.debug('results: %s', num_sec_results) results.update(num_sec_results) # Now that we've gotten all the back results, create a single # DatabaseReader to read all the fields. reader = DatabaseReader(fields, self.database, self.host, self.user, self.password) max_timestamp_seen = 0 while not self.quit_flag: # If we do have results, package them up and send them if results: send_message = json.dumps(results) logging.debug('Data server sending: %s', send_message) try: await self.websocket.send(send_message) except websockets.exceptions.ConnectionClosed: return # New results or not, take a nap before trying to fetch more results logging.debug('Sleeping %g seconds', self.interval) await asyncio.sleep(self.interval) # What's the timestamp of the most recent result we've seen? # Each value should be a list of (timestamp, value) pairs. Look # at the last timestamp in each value list. for field in results: last_timestamp = results[field][-1][0] max_timestamp_seen = max(max_timestamp_seen, last_timestamp) # Bug's corner case: if we didn't retrieve any data on the first # time through (because it was all too old), max_timestamp_seen # will be zero, causing us to retrieve *all* the data in the DB # on the next iteration. If we do find that max_timestamp_seen # is zero, set it to "now" to prevent this. if not max_timestamp_seen: max_timestamp_seen = now logging.debug('Results: %s', results) if len(results): logging.info('Received %d fields, max timestamp %f', len(results), max_timestamp_seen) # Check whether there are results newer than latest timestamp # we've already seen. results = reader.read_time_range(start_time=max_timestamp_seen + EPSILON)