def write(self, record): """Note: Assume record begins with a timestamp string.""" if record is None: return # First things first: get the date string from the record try: time_str = record.split(' ')[0] ts = timestamp.timestamp(time_str, time_format=self.time_format) date_str = timestamp.date_str(ts, date_format=self.date_format) logging.debug('LogfileWriter date_str: %s', date_str) except ValueError: logging.error('LogfileWriter.write() - bad record timestamp: %s', record) return # Is it time to create a new file to write to? if not self.writer or date_str != self.current_date: self.current_filename = self.filebase + '-' + date_str self.current_date = date_str logging.info('LogfileWriter opening new file: %s', self.current_filename) self.writer = TextFileWriter(self.current_filename, self.flush) logging.debug('LogfileWriter writing record: %s', record) self.writer.write(record)
def test_write(self): with tempfile.TemporaryDirectory() as tmpdirname: writer = TextFileWriter(tmpdirname + '/f') f = open(tmpdirname + '/f') for line in SAMPLE_DATA: writer.write(line) self.assertEqual(line, f.readline().strip())
def run(self): """Create the virtual port with socat and start feeding it records from the designated logfile.""" self.socat_thread = threading.Thread(target=self._run_socat) self.socat_thread.start() time.sleep(0.2) self.reader = LogfileReader(filebase=self.source_file, use_timestamps=self.use_timestamps) self.strip = SliceTransform('1:') # strip off the first field) self.writer = TextFileWriter(self.write_port, truncate=True) while not self.quit: record = self.reader.read() # get the next record logging.debug('SimSerial got: %s', record) if record is None: break record = self.strip.transform(record) # strip the timestamp if record: logging.debug('SimSerial writing: %s', record) self.writer.write(record) # and write it to the virtual port # If we're here, we got None from our input, and are done. Signal # for run_socat to exit self.quit = True
def run(self, loop=False): # If self.serial_params is None, it means that either read or # write device already exist, so we shouldn't actually run, or # we'll destroy them. if not self.serial_params: return """Create the virtual port with socat and start feeding it records from the designated logfile. If loop==True, loop when reaching end of input.""" self.socat_thread = threading.Thread(target=self._run_socat, daemon=True) self.socat_thread.start() time.sleep(0.2) self.reader = LogfileReader(filebase=self.filebase, use_timestamps=True, record_format=self.record_format, time_format=self.time_format) self.writer = TextFileWriter(self.write_port, truncate=True) logging.info('Starting %s: %s', self.read_port, self.filebase) while not self.quit: try: record = self.reader.read() # get the next record logging.debug('SimSerial got: %s', record) # End of input? If loop==True, re-open the logfile from the start if record is None: if not loop: break self.reader = LogfileReader( filebase=self.filebase, use_timestamps=True, record_format=self.record_format, time_format=self.time_format) # We've now got a record. Try parsing timestamp off it try: parsed_record = self.compiled_record_format.parse( record).named record = parsed_record['record'] # We had a problem parsing. Discard record and try reading next one. except (KeyError, ValueError, AttributeError): logging.warning('Unable to parse record into "%s"', self.record_format) logging.warning('Record: %s', record) continue if not record: continue logging.debug('SimSerial writing: %s', record) self.writer.write(record) # and write it to the virtual port except (OSError, KeyboardInterrupt): break # If we're here, we got None from our input, and are done. Signal # for run_socat to exit self.quit = True
def test_prefix(self): prefix_1 = PrefixTransform('p1') prefix_2 = PrefixTransform('p2') f1_name = self.tmpdirname + '/f1' f2_name = self.tmpdirname + '/f2' writer = ComposedWriter( transforms=[prefix_1, prefix_2], writers=[TextFileWriter(f1_name), TextFileWriter(f2_name)]) f1 = open(f1_name, 'r') f2 = open(f2_name, 'r') for line in SAMPLE_DATA: writer.write(line) time.sleep(0.1) f1_line = f1.readline().rstrip() f2_line = f2.readline().rstrip() logging.info('wrote: "%s", f1: "%s", f2: "%s"', line, f1_line, f2_line) self.assertEqual('p2 p1 ' + line, f1_line) self.assertEqual('p2 p1 ' + line, f2_line)
class LogfileWriter(Writer): """Write to the specified file. If filename is empty, write to stdout.""" def __init__(self, filebase=None, flush=True, time_format=timestamp.TIME_FORMAT, date_format=timestamp.DATE_FORMAT): """ Write timestamped text records to file. Base filename will have date appended, in keeping with R2R format recommendations (http://www.rvdata.us/operators/directory). When timestamped date on records rolls over to next day, create new file with new date suffix. filebase Base name of file to write to. Will have record date appended, e.g. flush If True (default), flush after every write() call date_fomat A strftime-compatible string, such as '%Y-%m-%d'; defaults to whatever's defined in utils.timestamps.DATE_FORMAT. """ super().__init__(input_format=Text) self.filebase = filebase self.flush = flush self.time_format = time_format self.date_format = date_format self.current_date = None self.current_filename = None self.writer = None ############################ def write(self, record): """Note: Assume record begins with a timestamp string.""" if record is None: return # First things first: get the date string from the record try: time_str = record.split(' ')[0] ts = timestamp.timestamp(time_str, time_format=self.time_format) date_str = timestamp.date_str(ts, date_format=self.date_format) logging.debug('LogfileWriter date_str: %s', date_str) except ValueError: logging.error('LogfileWriter.write() - bad record timestamp: %s', record) return # Is it time to create a new file to write to? if not self.writer or date_str != self.current_date: self.current_filename = self.filebase + '-' + date_str self.current_date = date_str logging.info('LogfileWriter opening new file: %s', self.current_filename) self.writer = TextFileWriter(self.current_filename, self.flush) logging.debug('LogfileWriter writing record: %s', record) self.writer.write(record)
def run(self, loop=False): # If self.serial_params is None, it means that either read or # write device already exist, so we shouldn't actually run, or # we'll destroy them. if not self.serial_params: return """Create the virtual port with socat and start feeding it records from the designated logfile. If loop==True, loop when reaching end of input.""" self.socat_thread = threading.Thread(target=self._run_socat, daemon=True) self.socat_thread.start() time.sleep(0.2) self.reader = LogfileReader(filebase=self.filebase, use_timestamps=True, time_format=self.time_format) self.slice_n = SliceTransform( '1:') # strip off the first field (timestamp) self.writer = TextFileWriter(self.write_port, truncate=True) logging.info('Starting %s: %s', self.read_port, self.filebase) while not self.quit: try: record = self.reader.read() # get the next record logging.debug('SimSerial got: %s', record) # End of input? If loop==True, re-open the logfile from the start if record is None: if not loop: break self.reader = LogfileReader(filebase=self.filebase, use_timestamps=True, time_format=self.time_format) record = self.slice_n.transform(record) # strip the timestamp if not record: continue if self.timestamp: # do we want to add a timestamp? record = self.timestamp.transform(record) if self.prefix: # do we want to add prefix? record = self.prefix.transform(record) logging.debug('SimSerial writing: %s', record) self.writer.write(record) # and write it to the virtual port except (OSError, KeyboardInterrupt): break # If we're here, we got None from our input, and are done. Signal # for run_socat to exit self.quit = True
def test_read_all_write_one(self): readers = [] for tmpfilename in self.tmpfilenames: readers.append(TextFileReader(tmpfilename, interval=0.2)) transforms = [PrefixTransform('prefix_1'), PrefixTransform('prefix_2')] outfilename = self.tmpdirname + '/f_out' writers = [TextFileWriter(outfilename)] listener = Listener(readers, transforms, writers) listener.run() out_lines = [] with open(outfilename, 'r') as f: for line in f.readlines(): out_lines.append(line.rstrip()) out_lines.sort() source_lines = [] for f in SAMPLE_DATA: source_lines.extend( ['prefix_2 prefix_1 ' + f for f in SAMPLE_DATA[f]]) source_lines.sort() logging.debug('out: %s, source: %s', out_lines, source_lines) self.assertEqual(out_lines, source_lines)
def test_basic(self): # Create a file temp_dir = tempfile.TemporaryDirectory() temp_dir_name = temp_dir.name test_file = temp_dir_name + '/test.txt' logging.info('creating temporary file "%s"', test_file) client_writer = TextFileWriter(filename=test_file) timeout_writer = TimeoutWriter(writer=client_writer, timeout=0.5, message='off', resume_message='on') time.sleep(0.75) # trigger an "off" message timeout_writer.write('foo') # trigger an "on" message time.sleep(1.2) # should trigger just one "off" message timeout_writer.quit() # Now check the file - make sure we got with open(test_file, 'r') as f: lines = f.read().strip().split('\n') self.assertEqual(lines[0], 'off') self.assertEqual(lines[1], 'on') self.assertEqual(lines[2], 'off') self.assertEqual(len(lines), 3)
def setUp(self): # To suppress resource warnings about unclosed files warnings.simplefilter("ignore", ResourceWarning) # Create a file self.temp_dir = tempfile.TemporaryDirectory() self.temp_dir_name = self.temp_dir.name self.source_name = self.temp_dir_name + '/source.txt' self.dest_name = self.temp_dir_name + '/dest.txt' # Create the source file writer = TextFileWriter(self.source_name) for line in SAMPLE_DATA: writer.write(line) self.config = CONFIG self.config['readers']['kwargs']['file_spec'] = self.source_name self.config['writers']['kwargs']['filename'] = self.dest_name
def test_check_format(self): f1_name = self.tmpdirname + '/f1' f2_name = self.tmpdirname + '/f2' # This should be okay ComposedWriter( transforms=[PrefixTransform('prefix')], writers=[TextFileWriter(f1_name), TextFileWriter(f2_name)], check_format=True) # Should raise an error if formats are not compatible with self.assertLogs(logging.getLogger(), logging.ERROR): with self.assertRaises(ValueError): ComposedWriter( transforms=[ParseNMEATransform()], writers=[TextFileWriter(f1_name), TextFileWriter(f2_name)], check_format=True)
def test_formats(self): writer = TextFileWriter(filename=None) self.assertEqual(writer.input_format(), formats.Text) self.assertEqual(writer.input_format(formats.NMEA), formats.NMEA) self.assertEqual(writer.input_format(), formats.NMEA) with self.assertRaises(TypeError): writer.input_format('not a format')
def write(self, record): """Note: Assume record begins with a timestamp string.""" if record is None: return # If we've got a list, hope it's a list of records. Recurse, # calling write() on each of the list elements in order. if type(record) is list: for single_record in record: self.write(single_record) return if not type(record) is str: logging.error( 'LogfileWriter.write() - record is not timestamped string: ' '%s', record) return # First things first: get the date string from the record try: time_str = record.split(self.split_char)[0] ts = timestamp.timestamp(time_str, time_format=self.time_format) hr_str = self.rollover_hourly and timestamp.date_str( ts, date_format='_%H00') or "" date_str = timestamp.date_str(ts, date_format=self.date_format) logging.debug('LogfileWriter date_str: %s', date_str) except ValueError: logging.error('LogfileWriter.write() - bad record timestamp: %s', record) return # Is it time to create a new file to write to? if not self.writer or date_str != self.current_date or hr_str != self.current_hour: self.current_filename = self.filebase + '-' + date_str + hr_str + self.suffix self.current_date = date_str self.current_hour = self.rollover_hourly and hr_str or "" logging.info('LogfileWriter opening new file: %s', self.current_filename) self.writer = TextFileWriter(self.current_filename, self.flush) logging.debug('LogfileWriter writing record: %s', record) self.writer.write(record)
def run(self, loop=False): """Create the virtual port with socat and start feeding it records from the designated logfile. If loop==True, loop when reaching end of input.""" self.socat_thread = threading.Thread(target=self._run_socat, daemon=True) self.socat_thread.start() time.sleep(0.2) self.reader = LogfileReader(filebase=self.source_file, use_timestamps=self.use_timestamps, time_format=self.time_format) self.strip = SliceTransform('1:') # strip off the first field) self.writer = TextFileWriter(self.write_port, truncate=True) while not self.quit: try: record = self.reader.read() # get the next record logging.debug('SimSerial got: %s', record) # End of input? If loop==True, re-open the logfile from the start if record is None: if not loop: break self.reader = LogfileReader( filebase=self.source_file, use_timestamps=self.use_timestamps) record = self.strip.transform(record) # strip the timestamp if record: logging.debug('SimSerial writing: %s', record) self.writer.write( record) # and write it to the virtual port except (OSError, KeyboardInterrupt): break # If we're here, we got None from our input, and are done. Signal # for run_socat to exit self.quit = True
def test_all_files(self): f1_name = self.tmpdirname + '/f1' f2_name = self.tmpdirname + '/f2' writer = ComposedWriter( transforms=[], writers=[TextFileWriter(f1_name), TextFileWriter(f2_name)]) f1 = open(f1_name, 'r') f2 = open(f2_name, 'r') for line in SAMPLE_DATA: writer.write(line) time.sleep(0.1) f1_line = f1.readline().rstrip() f2_line = f2.readline().rstrip() logging.info('wrote: "%s", f1: "%s", f2: "%s"', line, f1_line, f2_line) self.assertEqual(line, f1_line) self.assertEqual(line, f2_line)
def test_read_one_write_all(self): readers = TextFileReader(self.tmpfilenames[0]) outfilenames = [ self.tmpdirname + '/' + f for f in ['f1_out', 'f2_out', 'f3_out'] ] writers = [TextFileWriter(ofn) for ofn in outfilenames] listener = Listener(readers=readers, writers=writers) listener.run() for ofn in outfilenames: line_num = 0 with open(ofn, 'r') as f: for line in f.readlines(): self.assertEqual(SAMPLE_DATA['f1'][line_num], line.rstrip()) line_num += 1
class SimSerial: """Create a virtual serial port and feed stored logfile data to it.""" ############################ def __init__(self, port, source_file, use_timestamps=True, baudrate=9600, bytesize=8, parity='N', stopbits=1, timeout=None, xonxoff=False, rtscts=False, write_timeout=None, dsrdtr=False, inter_byte_timeout=None, exclusive=None): """Takes source file, whether to deliver data at rate indicated by timestamps, and the standard parameters that a serial port takes.""" self.source_file = source_file self.use_timestamps = use_timestamps # We'll create two virtual ports: 'port' and 'port_in'; we will write # to port_in and read the values back out from port self.read_port = port self.write_port = port + '_in' self.serial_params = { 'baudrate': baudrate, 'byteside': bytesize, 'parity': parity, 'stopbits': stopbits, 'timeout': timeout, 'xonxoff': xonxoff, 'rtscts': rtscts, 'write_timeout': write_timeout, 'dsrdtr': dsrdtr, 'inter_byte_timeout': inter_byte_timeout, 'exclusive': exclusive } self.quit = False # Finally, check that our needed 'socat' actually exists if not subprocess.run(['which', 'socat'], stdout=subprocess.PIPE).stdout: raise NameError( 'Executable "socat" not found on path. Please refer ' 'to installation guide to install socat.') ############################ def _run_socat(self): """Internal: run the actual command.""" verbose = '-d' write_port_params = 'pty,link=%s,raw,echo=0' % self.write_port read_port_params = 'pty,link=%s,raw,echo=0' % self.read_port cmd = [ '/usr/bin/env', 'socat', verbose, #verbose, # repeating makes it more verbose read_port_params, write_port_params, ] try: # Run socat process using Popen, checking every second or so whether # it's died (poll() != None) or we've gotten a quit signal. logging.info('Calling: %s', ' '.join(cmd)) socat_process = subprocess.Popen(cmd) while not self.quit and not socat_process.poll(): try: socat_process.wait(1) except subprocess.TimeoutExpired: pass except Exception as e: logging.error('ERROR: socat command: %s', e) # If here, process has terminated, or we've seen self.quit. We # want both to be true: if we've terminated, set self.quit so that # 'run' loop can exit. If self.quit, terminate process. if self.quit: socat_process.kill() else: self.quit = True logging.info('Finished: %s', ' '.join(cmd)) ############################ def run(self): """Create the virtual port with socat and start feeding it records from the designated logfile.""" self.socat_thread = threading.Thread(target=self._run_socat) self.socat_thread.start() time.sleep(0.2) self.reader = LogfileReader(filebase=self.source_file, use_timestamps=self.use_timestamps) self.strip = SliceTransform('1:') # strip off the first field) self.writer = TextFileWriter(self.write_port, truncate=True) while not self.quit: record = self.reader.read() # get the next record logging.debug('SimSerial got: %s', record) if record is None: break record = self.strip.transform(record) # strip the timestamp if record: logging.debug('SimSerial writing: %s', record) self.writer.write(record) # and write it to the virtual port # If we're here, we got None from our input, and are done. Signal # for run_socat to exit self.quit = True
class SimSerial: """Create a virtual serial port and feed stored logfile data to it.""" ############################ def __init__(self, port, prefix=None, timestamp=False, time_format=TIME_FORMAT, filebase=None, eol='\n', baudrate=9600, bytesize=8, parity='N', stopbits=1, timeout=None, xonxoff=False, rtscts=False, write_timeout=None, dsrdtr=False, inter_byte_timeout=None, exclusive=None): """ Simulate a serial port, feeding it data from the specified file. ``` port - Temporary serial port to create and make available for reading records. prefix - If non-empty, prefix name prefix to add timestamp = If True, apply current timestamp to record time_format - What format to use for timestamp ``` """ # We'll create two virtual ports: 'port' and 'port_in'; we will write # to port_in and read the values back out from port self.read_port = port self.write_port = port + '_in' self.prefix = PrefixTransform(prefix) if prefix else None self.timestamp = TimestampTransform() if timestamp else None self.time_format = time_format self.filebase = filebase self.serial_params = None # Complain, but go ahead if read_port or write_port exist. for path in [self.read_port, self.write_port]: if os.path.exists(path): logging.warning('Path %s exists; overwriting!') # Do we have any files we can actually read from? if not glob.glob(filebase + '*'): logging.warning('%s: no files matching "%s*"', prefix, filebase) return # Set up our parameters self.serial_params = {'baudrate': baudrate, 'byteside': bytesize, 'parity': parity, 'stopbits': stopbits, 'timeout': timeout, 'xonxoff': xonxoff, 'rtscts': rtscts, 'write_timeout': write_timeout, 'dsrdtr': dsrdtr, 'inter_byte_timeout': inter_byte_timeout, 'exclusive': exclusive} self.quit = False # Finally, find path to socat executable self.socat_path = None for socat_path in ['/usr/bin/socat', '/usr/local/bin/socat']: if os.path.exists(socat_path) and os.path.isfile(socat_path): self.socat_path = socat_path if not self.socat_path: raise NameError('Executable "socat" not found on path. Please refer ' 'to installation guide to install socat.') ############################ def _run_socat(self): """Internal: run the actual command.""" verbose = '-d' write_port_params = 'pty,link=%s,raw,echo=0' % self.write_port read_port_params = 'pty,link=%s,raw,echo=0' % self.read_port cmd = [self.socat_path, verbose, #verbose, # repeating makes it more verbose read_port_params, write_port_params, ] try: # Run socat process using Popen, checking every second or so whether # it's died (poll() != None) or we've gotten a quit signal. logging.info('Calling: %s', ' '.join(cmd)) socat_process = subprocess.Popen(cmd) while not self.quit and not socat_process.poll(): try: socat_process.wait(1) except subprocess.TimeoutExpired: pass except Exception as e: logging.error('ERROR: socat command: %s', e) # If here, process has terminated, or we've seen self.quit. We # want both to be true: if we've terminated, set self.quit so that # 'run' loop can exit. If self.quit, terminate process. if self.quit: socat_process.kill() else: self.quit = True logging.info('Finished: %s', ' '.join(cmd)) ############################ def run(self, loop=False): # If self.serial_params is None, it means that either read or # write device already exist, so we shouldn't actually run, or # we'll destroy them. if not self.serial_params: return """Create the virtual port with socat and start feeding it records from the designated logfile. If loop==True, loop when reaching end of input.""" self.socat_thread = threading.Thread(target=self._run_socat, daemon=True) self.socat_thread.start() time.sleep(0.2) self.reader = LogfileReader(filebase=self.filebase, use_timestamps=True, time_format=self.time_format) self.slice_n = SliceTransform('1:') # strip off the first field (timestamp) self.writer = TextFileWriter(self.write_port, truncate=True) logging.info('Starting %s: %s', self.read_port, self.filebase) while not self.quit: try: record = self.reader.read() # get the next record logging.debug('SimSerial got: %s', record) # End of input? If loop==True, re-open the logfile from the start if record is None: if not loop: break self.reader = LogfileReader(filebase=self.filebase, use_timestamps=True, time_format=self.time_format) record = self.slice_n.transform(record) # strip the timestamp if not record: continue if self.timestamp: # do we want to add a timestamp? record = self.timestamp.transform(record) if self.prefix: # do we want to add prefix? record = self.prefix.transform(record) logging.debug('SimSerial writing: %s', record) self.writer.write(record) # and write it to the virtual port except (OSError, KeyboardInterrupt): break # If we're here, we got None from our input, and are done. Signal # for run_socat to exit self.quit = True
def test_compatible(self): # Don't specify 'tail' and expect there to be no data with tempfile.TemporaryDirectory() as tmpdirname: writer = TextFileWriter(tmpdirname + '/f') reader = TextFileReader(tmpdirname + '/f') self.assertTrue(writer.can_accept(reader))
transforms.append(FromJSONTransform()) if new_args.from_json_to_das_record: transforms.append(FromJSONTransform(das_record=True)) if new_args.to_das_record: transforms.append( ToDASRecordTransform(field_name=new_args.to_das_record)) ########################## # Writers if new_args.write_file: for filename in new_args.write_file.split(','): if filename == '-': filename = None writers.append(TextFileWriter(filename=filename)) if new_args.write_logfile: writers.append(LogfileWriter(filebase=new_args.write_logfile)) if new_args.write_network: eol = all_args.network_eol for addr in new_args.write_network.split(','): writers.append(NetworkWriter(network=addr, eol=eol)) if new_args.write_udp: eol = all_args.network_eol for addr_str in new_args.write_udp.split(','): addr = addr_str.split(':') dest = '' interface = ''
dest='verbosity', default=0, action='count', help='Increase output verbosity') args = parser.parse_args() # Set logging verbosity LOGGING_FORMAT = '%(asctime)-15s %(filename)s:%(lineno)d %(message)s' LOG_LEVELS = {0: logging.WARNING, 1: logging.INFO, 2: logging.DEBUG} log_level = LOG_LEVELS[min(args.verbosity, max(LOG_LEVELS))] logging.basicConfig(format=LOGGING_FORMAT) logging.getLogger().setLevel(log_level) if args.stderr_file: stderr_writer = [ TextFileWriter(filename=args.stderr_file, split_by_date=True) ] logging.getLogger().addHandler(StdErrLoggingHandler(stderr_writer)) logging.info('Starting CachedDataServer') server = CachedDataServer(port=args.port, interval=args.interval, back_seconds=args.back_seconds, max_records=args.max_records, cleanup_interval=args.cleanup_interval, disk_cache=args.disk_cache) # Only create reader(s) if they've given us a network to read from; # otherwise, count on data coming from websocket publish # connections. if args.udp:
class LogfileWriter(Writer): """Write to the specified file. If filename is empty, write to stdout.""" def __init__(self, filebase=None, flush=True, time_format=timestamp.TIME_FORMAT, date_format=timestamp.DATE_FORMAT, split_char=' ', suffix='', rollover_hourly=False): """ Write timestamped text records to file. Base filename will have date appended, in keeping with R2R format recommendations (http://www.rvdata.us/operators/directory). When timestamped date on records rolls over to next day, create new file with new date suffix. ``` filebase Base name of file to write to. Will have record date appended, e.g. flush If True (default), flush after every write() call date_fomat A strftime-compatible string, such as '%Y-%m-%d'; defaults to whatever's defined in utils.timestamps.DATE_FORMAT. split_char Delimiter between timestamp and rest of message suffix string to apply to the end of the log filename rollover_hourly Set files to truncate by hour. By default files will truncate by day ``` """ super().__init__(input_format=Text) self.filebase = filebase self.flush = flush self.time_format = time_format self.date_format = date_format self.split_char = split_char self.suffix = suffix self.rollover_hourly = rollover_hourly self.current_date = None self.current_hour = None self.current_filename = None self.writer = None ############################ def write(self, record): """Note: Assume record begins with a timestamp string.""" if record is None: return # If we've got a list, hope it's a list of records. Recurse, # calling write() on each of the list elements in order. if type(record) is list: for single_record in record: self.write(single_record) return if not type(record) is str: logging.error( 'LogfileWriter.write() - record is not timestamped string: ' '%s', record) return # First things first: get the date string from the record try: time_str = record.split(self.split_char)[0] ts = timestamp.timestamp(time_str, time_format=self.time_format) hr_str = self.rollover_hourly and timestamp.date_str( ts, date_format='_%H00') or "" date_str = timestamp.date_str(ts, date_format=self.date_format) logging.debug('LogfileWriter date_str: %s', date_str) except ValueError: logging.error('LogfileWriter.write() - bad record timestamp: %s', record) return # Is it time to create a new file to write to? if not self.writer or date_str != self.current_date or hr_str != self.current_hour: self.current_filename = self.filebase + '-' + date_str + hr_str + self.suffix self.current_date = date_str self.current_hour = self.rollover_hourly and hr_str or "" logging.info('LogfileWriter opening new file: %s', self.current_filename) self.writer = TextFileWriter(self.current_filename, self.flush) logging.debug('LogfileWriter writing record: %s', record) self.writer.write(record)
'--verbosity', dest='verbosity', default=0, action='count', help='Increase output verbosity') args = parser.parse_args() LOGGING_FORMAT = '%(asctime)-15s %(message)s' logging.basicConfig(format=LOGGING_FORMAT) LOG_LEVELS = {0: logging.WARNING, 1: logging.INFO, 2: logging.DEBUG} args.verbosity = min(args.verbosity, max(LOG_LEVELS)) logging.getLogger().setLevel(LOG_LEVELS[args.verbosity]) reader = TextFileReader(file_spec=args.read, tail=args.tail) writer = TextFileWriter(filename=args.write) if args.prefix: prefix_transform = PrefixTransform(args.prefix) if args.timestamp: timestamp_transform = TimestampTransform() while True: record = reader.read() now = time.time() logging.info('Got record: %s', record) if record is None: if not args.tail:
action='count', help='Increase output verbosity') parser.add_argument('-V', '--logger_verbosity', dest='logger_verbosity', default=0, action='count', help='Increase output verbosity of component loggers') args = parser.parse_args() # Set up logging first of all LOG_LEVELS = {0: logging.WARNING, 1: logging.INFO, 2: logging.DEBUG} log_level = LOG_LEVELS[min(args.verbosity, max(LOG_LEVELS))] if args.stderr_file: stderr_writers = [TextFileWriter(args.stderr_file)] logging.getLogger().addHandler(StdErrLoggingHandler(stderr_writers)) # What level do we want our component loggers to write? logger_log_level = LOG_LEVELS[min(args.logger_verbosity, max(LOG_LEVELS))] configs = read_config(args.config) if args.config else None # If they've specified a mode, see if what we've read in is in fact # a cruise definition with modes we can choose from. if args.mode: modes = configs.get('modes', None) if not modes: logging.fatal('--mode specified, but "%s" has no modes' % args.config) sys.exit(1)