def test_basic(self):
dr = DASRecord()
self.assertAlmostEqual(dr.timestamp, timestamp.timestamp(), 2)
self.assertAlmostEqual(dr.timestamp, timestamp.timestamp(), 2)
self.assertEqual(dr.data_id, None)
self.assertEqual(dr.fields, {})
self.assertEqual(dr.metadata, {})
def test_default(self):
transform = TimestampTransform()
self.assertIsNone(transform.transform(None))
result = transform.transform('blah')
time_str = result.split()[0]
then = timestamp.timestamp(time_str=time_str)
now = timestamp.timestamp()
self.assertAlmostEqual(then, now, places=1)
self.assertEqual(result.split()[1], 'blah')
def read(self):
"""
Return the next line in the file(s), or None if there are no more
records (as opposed to '' if the next record is a blank line). To test
EOF you'll need to test
if record is None:
no more records...
rather than simply
if not record:
could be EOF or simply an empty next line
"""
record = self.reader.read()
if not record:
return None
# NOTE: It feels like we should check here that the reader's
# current file really does match our logfile name format...
if self.use_timestamps:
# Get timestamp off record
time_str = record.split(' ', 1)[0]
ts = timestamp.timestamp(time_str, time_format=self.time_format)
# If we're not trying to return records in intervals that match
# their original intervals, we're done - just return it.
if not self.use_timestamps:
self.prev_record = record
# We need this in case the next call is seek_time() or read_time_range().
# This is less expensive than parsing every timestamp and keeping
# self.last_timestamp, but an alternative might be to implement
# read_previous(), which would be expensive but which could be called
# only when actually needed.
return record
desired_interval = ts - self.last_timestamp
now = timestamp.timestamp()
actual_interval = now - self.last_read
logging.debug('Desired interval %f, actual %f; sleeping %f',
desired_interval, actual_interval,
max(0, desired_interval - actual_interval))
time.sleep(max(0, desired_interval - actual_interval))
self.last_timestamp = ts
self.last_read = timestamp.timestamp()
self.prev_record = record
return record
def write(self, record):
"""Note: Assume record begins with a timestamp string."""
if record is None:
return
# First things first: get the date string from the record
try:
time_str = record.split(' ')[0]
ts = timestamp.timestamp(time_str, time_format=self.time_format)
date_str = timestamp.date_str(ts, date_format=self.date_format)
logging.debug('LogfileWriter date_str: %s', date_str)
except ValueError:
logging.error('LogfileWriter.write() - bad record timestamp: %s',
record)
return
# Is it time to create a new file to write to?
if not self.writer or date_str != self.current_date:
self.current_filename = self.filebase + '-' + date_str
self.current_date = date_str
logging.info('LogfileWriter opening new file: %s',
self.current_filename)
self.writer = TextFileWriter(self.current_filename, self.flush)
logging.debug('LogfileWriter writing record: %s', record)
self.writer.write(record)
def test_list(self):
transform = TimestampTransform()
self.assertIsNone(transform.transform(None))
record = ['foo', 'bar', 'baz']
result = transform.transform(record)
timestamps = [r.split()[0] for r in result]
self.assertEqual(timestamps[0], timestamps[1])
self.assertEqual(timestamps[1], timestamps[2])
then = timestamp.timestamp(time_str=timestamps[0])
now = timestamp.timestamp()
self.assertAlmostEqual(then, now, places=1)
sources = [r.split()[1] for r in result]
self.assertEqual(sources, record)
def parse_record(self, nmea_record):
"""Receive an id-prefixed, timestamped NMEA record."""
if not nmea_record:
return None
if not type(nmea_record) == type(''):
logging.error('Record is not NMEA string: "%s"', nmea_record)
return None
try:
(data_id, raw_ts, message) = nmea_record.strip().split(sep=' ',
maxsplit=2)
ts = timestamp(raw_ts)
except ValueError:
logging.error(
'Record not in <data_id> <timestamp> <NMEA> format: "%s"',
nmea_record)
return None
# Figure out what kind of message we're expecting, based on data_id
sensor = self.sensors.get(data_id, None)
if not sensor:
logging.error('Unrecognized data_id ("%s") in record: %s', data_id,
nmea_record)
return None
model_name = sensor.get('model', None)
if not model_name:
logging.error('No "model" for sensor %s', sensor)
return None
# If something goes wrong during parsing, we'll get a ValueError
try:
(fields,
message_type) = self.parse_nmea(sensor_model_name=model_name,
message=message)
except ValueError as e:
logging.error(str(e))
return None
# Finally, convert field values to variable names specific to sensor
sensor_fields = sensor.get('fields', None)
if not sensor_fields:
logging.error('No "fields" definition found for sensor %s',
data_id)
return None
named_fields = {}
for field_name in fields:
var_name = sensor_fields.get(field_name, None)
if var_name:
named_fields[var_name] = fields[field_name]
record = DASRecord(data_id=data_id,
message_type=message_type,
timestamp=ts,
fields=named_fields)
logging.debug('created DASRecord: %s', str(record))
return record
def read(self):
"""
Return the next line in the file(s), or None if there are no more
records (as opposed to '' if the next record is a blank line). To test
EOF you'll need to test
if record is None:
no more records...
rather than simply
if not record:
could be EOF or simply an empty next line
"""
record = self.reader.read()
if not record:
return None
# NOTE: It feels like we should check here that the reader's
# current file really does match our logfile name format...
# If we're not trying to return records in intervals that match
# their original intervals, we're done - just return it.
if not self.use_timestamps:
return record
# Get timestamp off record
time_str = record.split(' ', 1)[0]
ts = timestamp.timestamp(time_str)
desired_interval = ts - self.last_timestamp
now = timestamp.timestamp()
actual_interval = now - self.last_read
logging.debug('Desired interval %f, actual %f; sleeping %f',
desired_interval, actual_interval,
max(0, desired_interval - actual_interval))
time.sleep(max(0, desired_interval - actual_interval))
self.last_timestamp = ts
self.last_read = timestamp.timestamp()
return record
def write(self, record):
"""Note: Assume record begins with a timestamp string."""
if record is None:
return
# If we've got a list, hope it's a list of records. Recurse,
# calling write() on each of the list elements in order.
if type(record) is list:
for single_record in record:
self.write(single_record)
return
if not type(record) is str:
logging.error(
'LogfileWriter.write() - record not timestamped: %s ', record)
return
# Get the timestamp we'll be using
try: # Try to extract timestamp from record
time_str = record.split(self.split_char)[0]
ts = timestamp.timestamp(time_str, time_format=self.time_format)
except ValueError:
logging.error('LogfileWriter.write() - bad timestamp: %s', record)
return
# Now parse ts into hour and date strings
hr_str = self.rollover_hourly and \
timestamp.date_str(ts, date_format='_%H00') or ""
date_str = timestamp.date_str(ts, date_format=self.date_format)
logging.debug('LogfileWriter date_str: %s', date_str)
# Is it time to create a new file to write to?
if not self.writer or date_str != self.current_date or hr_str != self.current_hour:
self.current_filename = self.filebase + '-' + date_str + hr_str + self.suffix
self.current_date = date_str
self.current_hour = self.rollover_hourly and hr_str or ""
logging.info('LogfileWriter opening new file: %s',
self.current_filename)
self.writer = FileWriter(filename=self.current_filename,
flush=self.flush)
logging.debug('LogfileWriter writing record: %s', record)
self.writer.write(record)
def write(self, record):
"""Note: Assume record begins with a timestamp string."""
if record is None:
return
# If we've got a list, hope it's a list of records. Recurse,
# calling write() on each of the list elements in order.
if type(record) is list:
for single_record in record:
self.write(single_record)
return
if not type(record) is str:
logging.error(
'LogfileWriter.write() - record is not timestamped string: '
'%s', record)
return
# First things first: get the date string from the record
try:
time_str = record.split()[0]
ts = timestamp.timestamp(time_str, time_format=self.time_format)
hr_str = self.rollover_hourly and timestamp.date_str(
ts, date_format='_%H00') or ""
date_str = timestamp.date_str(ts, date_format=self.date_format)
logging.debug('LogfileWriter date_str: %s', date_str)
except ValueError:
logging.error('LogfileWriter.write() - bad record timestamp: %s',
record)
return
# Is it time to create a new file to write to?
if not self.writer or date_str != self.current_date or hr_str != self.current_hour:
self.current_filename = self.filebase + '-' + date_str + hr_str + self.suffix
self.current_date = date_str
self.current_hour = self.rollover_hourly and hr_str or ""
logging.info('LogfileWriter opening new file: %s',
self.current_filename)
self.writer = TextFileWriter(self.current_filename, self.flush)
logging.debug('LogfileWriter writing record: %s', record)
self.writer.write(record)
def get_msec_timestamp(record):
time_str = record.split(' ', 1)[0]
return timestamp.timestamp(time_str,
time_format=timestamp.TIME_FORMAT) * 1000
def read(self):
"""
Return the next line in the file(s), or None if there are no more
records (as opposed to '' if the next record is a blank line). To test
EOF you'll need to test
if record is None:
no more records...
rather than simply
if not record:
could be EOF or simply an empty next line
"""
# NOTE: It feels like we should check here that the reader's
# current file really does match our logfile name format...
while True:
record = self.reader.read()
if not record: # None means we're out of records
return None
# If we've got a record and we're not using timestamps, we're
# done - just return it.
if not self.use_timestamps:
self.prev_record = record
# We need this in case the next call is seek_time() or
# read_time_range(). This is less expensive than parsing every
# timestamp and keeping self.last_timestamp, but an
# alternative might be to implement read_previous(), which
# would be expensive but which could be called only when
# actually needed.
return record
# If we are using timestamps, make sure we can parse the
# timestamp off the front. If we can't, complain and try getting
# the next record.
try:
time_str = record.split(' ', 1)[0]
ts = timestamp.timestamp(time_str,
time_format=self.time_format)
break
except ValueError:
# If, for some reason, the record is malformed, complain and
# loop to try fetching the next record
logging.warning('Unable to parse time string from record: %s',
record)
# If here, we've got a record and a timestamp and are intending to
# use it. Figure out how long we should sleep before returning it.
desired_interval = ts - self.last_timestamp
now = timestamp.timestamp()
actual_interval = now - self.last_read
logging.debug('Desired interval %f, actual %f; sleeping %f',
desired_interval, actual_interval,
max(0, desired_interval - actual_interval))
time.sleep(max(0, desired_interval - actual_interval))
self.last_timestamp = ts
self.last_read = timestamp.timestamp()
self.prev_record = record
return record
def _get_msec_timestamp(self, record):
time_str = record.split(' ', 1)[0]
return timestamp.timestamp(time_str,
time_format=self.time_format) * 1000
def read(self):
"""
Return the next line in the file(s), or None if there are no more
records (as opposed to '' if the next record is a blank line). To test
EOF you'll need to test
if record is None:
no more records...
rather than simply
if not record:
could be EOF or simply an empty next line
"""
# NOTE: It feels like we should check here that the reader's
# current file really does match our logfile name format...
while True:
record = self.reader.read()
if not record: # None means we're out of records
return None
# If we've got a record and we're not using timestamps, we're
# done - just return it.
if not self.use_timestamps:
self.prev_record = record
# We need this in case the next call is seek_time() or
# read_time_range(). This is less expensive than parsing every
# timestamp and keeping self.last_timestamp, but an
# alternative might be to implement read_previous(), which
# would be expensive but which could be called only when
# actually needed.
return record
# If we are using timestamps, make sure we can parse the
# timestamp off the front. If we can't, complain and try getting
# the next record.
try:
parsed_record = self.compiled_record_format.parse(record).named
ts = parsed_record['timestamp'].timestamp()
break
# We had a problem parsing. Discard record and try reading next one.
# Complain if appropriate.
except (KeyError, ValueError, AttributeError):
if not self.quiet:
logging.warning('Unable to parse record into "%s"',
self.record_format)
logging.warning('Record: %s', record)
continue
# If here, we've got a record and a timestamp and are intending to
# use it. Figure out how long we should sleep before returning it.
desired_interval = ts - self.last_timestamp
now = timestamp.timestamp()
actual_interval = now - self.last_read
logging.debug('Desired interval %f, actual %f; sleeping %f',
desired_interval, actual_interval,
max(0, desired_interval - actual_interval))
time.sleep(max(0, desired_interval - actual_interval))
self.last_timestamp = ts
self.last_read = timestamp.timestamp()
self.prev_record = record
return record
def test_timestamp(self):
self.assertAlmostEqual(timestamp.timestamp(timestamp.time_str()),
timestamp.timestamp(),
places=1)
self.assertEqual(timestamp.timestamp('1970-01-01T00:00:10.0Z'), 10.0)
