def test_basic(self): dr = DASRecord() self.assertAlmostEqual(dr.timestamp, timestamp.timestamp(), 2) self.assertAlmostEqual(dr.timestamp, timestamp.timestamp(), 2) self.assertEqual(dr.data_id, None) self.assertEqual(dr.fields, {}) self.assertEqual(dr.metadata, {})
def test_default(self): transform = TimestampTransform() self.assertIsNone(transform.transform(None)) result = transform.transform('blah') time_str = result.split()[0] then = timestamp.timestamp(time_str=time_str) now = timestamp.timestamp() self.assertAlmostEqual(then, now, places=1) self.assertEqual(result.split()[1], 'blah')
def read(self): """ Return the next line in the file(s), or None if there are no more records (as opposed to '' if the next record is a blank line). To test EOF you'll need to test if record is None: no more records... rather than simply if not record: could be EOF or simply an empty next line """ record = self.reader.read() if not record: return None # NOTE: It feels like we should check here that the reader's # current file really does match our logfile name format... if self.use_timestamps: # Get timestamp off record time_str = record.split(' ', 1)[0] ts = timestamp.timestamp(time_str, time_format=self.time_format) # If we're not trying to return records in intervals that match # their original intervals, we're done - just return it. if not self.use_timestamps: self.prev_record = record # We need this in case the next call is seek_time() or read_time_range(). # This is less expensive than parsing every timestamp and keeping # self.last_timestamp, but an alternative might be to implement # read_previous(), which would be expensive but which could be called # only when actually needed. return record desired_interval = ts - self.last_timestamp now = timestamp.timestamp() actual_interval = now - self.last_read logging.debug('Desired interval %f, actual %f; sleeping %f', desired_interval, actual_interval, max(0, desired_interval - actual_interval)) time.sleep(max(0, desired_interval - actual_interval)) self.last_timestamp = ts self.last_read = timestamp.timestamp() self.prev_record = record return record
def write(self, record): """Note: Assume record begins with a timestamp string.""" if record is None: return # First things first: get the date string from the record try: time_str = record.split(' ')[0] ts = timestamp.timestamp(time_str, time_format=self.time_format) date_str = timestamp.date_str(ts, date_format=self.date_format) logging.debug('LogfileWriter date_str: %s', date_str) except ValueError: logging.error('LogfileWriter.write() - bad record timestamp: %s', record) return # Is it time to create a new file to write to? if not self.writer or date_str != self.current_date: self.current_filename = self.filebase + '-' + date_str self.current_date = date_str logging.info('LogfileWriter opening new file: %s', self.current_filename) self.writer = TextFileWriter(self.current_filename, self.flush) logging.debug('LogfileWriter writing record: %s', record) self.writer.write(record)
def test_list(self): transform = TimestampTransform() self.assertIsNone(transform.transform(None)) record = ['foo', 'bar', 'baz'] result = transform.transform(record) timestamps = [r.split()[0] for r in result] self.assertEqual(timestamps[0], timestamps[1]) self.assertEqual(timestamps[1], timestamps[2]) then = timestamp.timestamp(time_str=timestamps[0]) now = timestamp.timestamp() self.assertAlmostEqual(then, now, places=1) sources = [r.split()[1] for r in result] self.assertEqual(sources, record)
def parse_record(self, nmea_record): """Receive an id-prefixed, timestamped NMEA record.""" if not nmea_record: return None if not type(nmea_record) == type(''): logging.error('Record is not NMEA string: "%s"', nmea_record) return None try: (data_id, raw_ts, message) = nmea_record.strip().split(sep=' ', maxsplit=2) ts = timestamp(raw_ts) except ValueError: logging.error( 'Record not in <data_id> <timestamp> <NMEA> format: "%s"', nmea_record) return None # Figure out what kind of message we're expecting, based on data_id sensor = self.sensors.get(data_id, None) if not sensor: logging.error('Unrecognized data_id ("%s") in record: %s', data_id, nmea_record) return None model_name = sensor.get('model', None) if not model_name: logging.error('No "model" for sensor %s', sensor) return None # If something goes wrong during parsing, we'll get a ValueError try: (fields, message_type) = self.parse_nmea(sensor_model_name=model_name, message=message) except ValueError as e: logging.error(str(e)) return None # Finally, convert field values to variable names specific to sensor sensor_fields = sensor.get('fields', None) if not sensor_fields: logging.error('No "fields" definition found for sensor %s', data_id) return None named_fields = {} for field_name in fields: var_name = sensor_fields.get(field_name, None) if var_name: named_fields[var_name] = fields[field_name] record = DASRecord(data_id=data_id, message_type=message_type, timestamp=ts, fields=named_fields) logging.debug('created DASRecord: %s', str(record)) return record
def read(self): """ Return the next line in the file(s), or None if there are no more records (as opposed to '' if the next record is a blank line). To test EOF you'll need to test if record is None: no more records... rather than simply if not record: could be EOF or simply an empty next line """ record = self.reader.read() if not record: return None # NOTE: It feels like we should check here that the reader's # current file really does match our logfile name format... # If we're not trying to return records in intervals that match # their original intervals, we're done - just return it. if not self.use_timestamps: return record # Get timestamp off record time_str = record.split(' ', 1)[0] ts = timestamp.timestamp(time_str) desired_interval = ts - self.last_timestamp now = timestamp.timestamp() actual_interval = now - self.last_read logging.debug('Desired interval %f, actual %f; sleeping %f', desired_interval, actual_interval, max(0, desired_interval - actual_interval)) time.sleep(max(0, desired_interval - actual_interval)) self.last_timestamp = ts self.last_read = timestamp.timestamp() return record
def write(self, record): """Note: Assume record begins with a timestamp string.""" if record is None: return # If we've got a list, hope it's a list of records. Recurse, # calling write() on each of the list elements in order. if type(record) is list: for single_record in record: self.write(single_record) return if not type(record) is str: logging.error( 'LogfileWriter.write() - record not timestamped: %s ', record) return # Get the timestamp we'll be using try: # Try to extract timestamp from record time_str = record.split(self.split_char)[0] ts = timestamp.timestamp(time_str, time_format=self.time_format) except ValueError: logging.error('LogfileWriter.write() - bad timestamp: %s', record) return # Now parse ts into hour and date strings hr_str = self.rollover_hourly and \ timestamp.date_str(ts, date_format='_%H00') or "" date_str = timestamp.date_str(ts, date_format=self.date_format) logging.debug('LogfileWriter date_str: %s', date_str) # Is it time to create a new file to write to? if not self.writer or date_str != self.current_date or hr_str != self.current_hour: self.current_filename = self.filebase + '-' + date_str + hr_str + self.suffix self.current_date = date_str self.current_hour = self.rollover_hourly and hr_str or "" logging.info('LogfileWriter opening new file: %s', self.current_filename) self.writer = FileWriter(filename=self.current_filename, flush=self.flush) logging.debug('LogfileWriter writing record: %s', record) self.writer.write(record)
def write(self, record): """Note: Assume record begins with a timestamp string.""" if record is None: return # If we've got a list, hope it's a list of records. Recurse, # calling write() on each of the list elements in order. if type(record) is list: for single_record in record: self.write(single_record) return if not type(record) is str: logging.error( 'LogfileWriter.write() - record is not timestamped string: ' '%s', record) return # First things first: get the date string from the record try: time_str = record.split()[0] ts = timestamp.timestamp(time_str, time_format=self.time_format) hr_str = self.rollover_hourly and timestamp.date_str( ts, date_format='_%H00') or "" date_str = timestamp.date_str(ts, date_format=self.date_format) logging.debug('LogfileWriter date_str: %s', date_str) except ValueError: logging.error('LogfileWriter.write() - bad record timestamp: %s', record) return # Is it time to create a new file to write to? if not self.writer or date_str != self.current_date or hr_str != self.current_hour: self.current_filename = self.filebase + '-' + date_str + hr_str + self.suffix self.current_date = date_str self.current_hour = self.rollover_hourly and hr_str or "" logging.info('LogfileWriter opening new file: %s', self.current_filename) self.writer = TextFileWriter(self.current_filename, self.flush) logging.debug('LogfileWriter writing record: %s', record) self.writer.write(record)
def get_msec_timestamp(record): time_str = record.split(' ', 1)[0] return timestamp.timestamp(time_str, time_format=timestamp.TIME_FORMAT) * 1000
def read(self): """ Return the next line in the file(s), or None if there are no more records (as opposed to '' if the next record is a blank line). To test EOF you'll need to test if record is None: no more records... rather than simply if not record: could be EOF or simply an empty next line """ # NOTE: It feels like we should check here that the reader's # current file really does match our logfile name format... while True: record = self.reader.read() if not record: # None means we're out of records return None # If we've got a record and we're not using timestamps, we're # done - just return it. if not self.use_timestamps: self.prev_record = record # We need this in case the next call is seek_time() or # read_time_range(). This is less expensive than parsing every # timestamp and keeping self.last_timestamp, but an # alternative might be to implement read_previous(), which # would be expensive but which could be called only when # actually needed. return record # If we are using timestamps, make sure we can parse the # timestamp off the front. If we can't, complain and try getting # the next record. try: time_str = record.split(' ', 1)[0] ts = timestamp.timestamp(time_str, time_format=self.time_format) break except ValueError: # If, for some reason, the record is malformed, complain and # loop to try fetching the next record logging.warning('Unable to parse time string from record: %s', record) # If here, we've got a record and a timestamp and are intending to # use it. Figure out how long we should sleep before returning it. desired_interval = ts - self.last_timestamp now = timestamp.timestamp() actual_interval = now - self.last_read logging.debug('Desired interval %f, actual %f; sleeping %f', desired_interval, actual_interval, max(0, desired_interval - actual_interval)) time.sleep(max(0, desired_interval - actual_interval)) self.last_timestamp = ts self.last_read = timestamp.timestamp() self.prev_record = record return record
def _get_msec_timestamp(self, record): time_str = record.split(' ', 1)[0] return timestamp.timestamp(time_str, time_format=self.time_format) * 1000
def read(self): """ Return the next line in the file(s), or None if there are no more records (as opposed to '' if the next record is a blank line). To test EOF you'll need to test if record is None: no more records... rather than simply if not record: could be EOF or simply an empty next line """ # NOTE: It feels like we should check here that the reader's # current file really does match our logfile name format... while True: record = self.reader.read() if not record: # None means we're out of records return None # If we've got a record and we're not using timestamps, we're # done - just return it. if not self.use_timestamps: self.prev_record = record # We need this in case the next call is seek_time() or # read_time_range(). This is less expensive than parsing every # timestamp and keeping self.last_timestamp, but an # alternative might be to implement read_previous(), which # would be expensive but which could be called only when # actually needed. return record # If we are using timestamps, make sure we can parse the # timestamp off the front. If we can't, complain and try getting # the next record. try: parsed_record = self.compiled_record_format.parse(record).named ts = parsed_record['timestamp'].timestamp() break # We had a problem parsing. Discard record and try reading next one. # Complain if appropriate. except (KeyError, ValueError, AttributeError): if not self.quiet: logging.warning('Unable to parse record into "%s"', self.record_format) logging.warning('Record: %s', record) continue # If here, we've got a record and a timestamp and are intending to # use it. Figure out how long we should sleep before returning it. desired_interval = ts - self.last_timestamp now = timestamp.timestamp() actual_interval = now - self.last_read logging.debug('Desired interval %f, actual %f; sleeping %f', desired_interval, actual_interval, max(0, desired_interval - actual_interval)) time.sleep(max(0, desired_interval - actual_interval)) self.last_timestamp = ts self.last_read = timestamp.timestamp() self.prev_record = record return record
def test_timestamp(self): self.assertAlmostEqual(timestamp.timestamp(timestamp.time_str()), timestamp.timestamp(), places=1) self.assertEqual(timestamp.timestamp('1970-01-01T00:00:10.0Z'), 10.0)