def write(self, record):
"""Note: Assume record begins with a timestamp string."""
if record is None:
return
# First things first: get the date string from the record
try:
time_str = record.split(' ')[0]
ts = timestamp.timestamp(time_str, time_format=self.time_format)
date_str = timestamp.date_str(ts, date_format=self.date_format)
logging.debug('LogfileWriter date_str: %s', date_str)
except ValueError:
logging.error('LogfileWriter.write() - bad record timestamp: %s',
record)
return
# Is it time to create a new file to write to?
if not self.writer or date_str != self.current_date:
self.current_filename = self.filebase + '-' + date_str
self.current_date = date_str
logging.info('LogfileWriter opening new file: %s',
self.current_filename)
self.writer = TextFileWriter(self.current_filename, self.flush)
logging.debug('LogfileWriter writing record: %s', record)
self.writer.write(record)
def test_write(self):
with tempfile.TemporaryDirectory() as tmpdirname:
writer = TextFileWriter(tmpdirname + '/f')
f = open(tmpdirname + '/f')
for line in SAMPLE_DATA:
writer.write(line)
self.assertEqual(line, f.readline().strip())
def run(self):
"""Create the virtual port with socat and start feeding it records from
the designated logfile."""
self.socat_thread = threading.Thread(target=self._run_socat)
self.socat_thread.start()
time.sleep(0.2)
self.reader = LogfileReader(filebase=self.source_file,
use_timestamps=self.use_timestamps)
self.strip = SliceTransform('1:') # strip off the first field)
self.writer = TextFileWriter(self.write_port, truncate=True)
while not self.quit:
record = self.reader.read() # get the next record
logging.debug('SimSerial got: %s', record)
if record is None:
break
record = self.strip.transform(record) # strip the timestamp
if record:
logging.debug('SimSerial writing: %s', record)
self.writer.write(record) # and write it to the virtual port
# If we're here, we got None from our input, and are done. Signal
# for run_socat to exit
self.quit = True
def run(self, loop=False):
# If self.serial_params is None, it means that either read or
# write device already exist, so we shouldn't actually run, or
# we'll destroy them.
if not self.serial_params:
return
"""Create the virtual port with socat and start feeding it records from
the designated logfile. If loop==True, loop when reaching end of input."""
self.socat_thread = threading.Thread(target=self._run_socat,
daemon=True)
self.socat_thread.start()
time.sleep(0.2)
self.reader = LogfileReader(filebase=self.filebase,
use_timestamps=True,
record_format=self.record_format,
time_format=self.time_format)
self.writer = TextFileWriter(self.write_port, truncate=True)
logging.info('Starting %s: %s', self.read_port, self.filebase)
while not self.quit:
try:
record = self.reader.read() # get the next record
logging.debug('SimSerial got: %s', record)
# End of input? If loop==True, re-open the logfile from the start
if record is None:
if not loop:
break
self.reader = LogfileReader(
filebase=self.filebase,
use_timestamps=True,
record_format=self.record_format,
time_format=self.time_format)
# We've now got a record. Try parsing timestamp off it
try:
parsed_record = self.compiled_record_format.parse(
record).named
record = parsed_record['record']
# We had a problem parsing. Discard record and try reading next one.
except (KeyError, ValueError, AttributeError):
logging.warning('Unable to parse record into "%s"',
self.record_format)
logging.warning('Record: %s', record)
continue
if not record:
continue
logging.debug('SimSerial writing: %s', record)
self.writer.write(record) # and write it to the virtual port
except (OSError, KeyboardInterrupt):
break
# If we're here, we got None from our input, and are done. Signal
# for run_socat to exit
self.quit = True
def test_prefix(self):
prefix_1 = PrefixTransform('p1')
prefix_2 = PrefixTransform('p2')
f1_name = self.tmpdirname + '/f1'
f2_name = self.tmpdirname + '/f2'
writer = ComposedWriter(
transforms=[prefix_1, prefix_2],
writers=[TextFileWriter(f1_name),
TextFileWriter(f2_name)])
f1 = open(f1_name, 'r')
f2 = open(f2_name, 'r')
for line in SAMPLE_DATA:
writer.write(line)
time.sleep(0.1)
f1_line = f1.readline().rstrip()
f2_line = f2.readline().rstrip()
logging.info('wrote: "%s", f1: "%s", f2: "%s"', line, f1_line,
f2_line)
self.assertEqual('p2 p1 ' + line, f1_line)
self.assertEqual('p2 p1 ' + line, f2_line)
class LogfileWriter(Writer):
"""Write to the specified file. If filename is empty, write to stdout."""
def __init__(self,
filebase=None,
flush=True,
time_format=timestamp.TIME_FORMAT,
date_format=timestamp.DATE_FORMAT):
"""
Write timestamped text records to file. Base filename will have
date appended, in keeping with R2R format recommendations
(http://www.rvdata.us/operators/directory). When timestamped date on
records rolls over to next day, create new file with new date suffix.
filebase Base name of file to write to. Will have record date
appended, e.g.
flush If True (default), flush after every write() call
date_fomat A strftime-compatible string, such as '%Y-%m-%d'; defaults
to whatever's defined in utils.timestamps.DATE_FORMAT.
"""
super().__init__(input_format=Text)
self.filebase = filebase
self.flush = flush
self.time_format = time_format
self.date_format = date_format
self.current_date = None
self.current_filename = None
self.writer = None
############################
def write(self, record):
"""Note: Assume record begins with a timestamp string."""
if record is None:
return
# First things first: get the date string from the record
try:
time_str = record.split(' ')[0]
ts = timestamp.timestamp(time_str, time_format=self.time_format)
date_str = timestamp.date_str(ts, date_format=self.date_format)
logging.debug('LogfileWriter date_str: %s', date_str)
except ValueError:
logging.error('LogfileWriter.write() - bad record timestamp: %s',
record)
return
# Is it time to create a new file to write to?
if not self.writer or date_str != self.current_date:
self.current_filename = self.filebase + '-' + date_str
self.current_date = date_str
logging.info('LogfileWriter opening new file: %s',
self.current_filename)
self.writer = TextFileWriter(self.current_filename, self.flush)
logging.debug('LogfileWriter writing record: %s', record)
self.writer.write(record)
def run(self, loop=False):
# If self.serial_params is None, it means that either read or
# write device already exist, so we shouldn't actually run, or
# we'll destroy them.
if not self.serial_params:
return
"""Create the virtual port with socat and start feeding it records from
the designated logfile. If loop==True, loop when reaching end of input."""
self.socat_thread = threading.Thread(target=self._run_socat,
daemon=True)
self.socat_thread.start()
time.sleep(0.2)
self.reader = LogfileReader(filebase=self.filebase,
use_timestamps=True,
time_format=self.time_format)
self.slice_n = SliceTransform(
'1:') # strip off the first field (timestamp)
self.writer = TextFileWriter(self.write_port, truncate=True)
logging.info('Starting %s: %s', self.read_port, self.filebase)
while not self.quit:
try:
record = self.reader.read() # get the next record
logging.debug('SimSerial got: %s', record)
# End of input? If loop==True, re-open the logfile from the start
if record is None:
if not loop:
break
self.reader = LogfileReader(filebase=self.filebase,
use_timestamps=True,
time_format=self.time_format)
record = self.slice_n.transform(record) # strip the timestamp
if not record:
continue
if self.timestamp: # do we want to add a timestamp?
record = self.timestamp.transform(record)
if self.prefix: # do we want to add prefix?
record = self.prefix.transform(record)
logging.debug('SimSerial writing: %s', record)
self.writer.write(record) # and write it to the virtual port
except (OSError, KeyboardInterrupt):
break
# If we're here, we got None from our input, and are done. Signal
# for run_socat to exit
self.quit = True
def test_read_all_write_one(self):
readers = []
for tmpfilename in self.tmpfilenames:
readers.append(TextFileReader(tmpfilename, interval=0.2))
transforms = [PrefixTransform('prefix_1'), PrefixTransform('prefix_2')]
outfilename = self.tmpdirname + '/f_out'
writers = [TextFileWriter(outfilename)]
listener = Listener(readers, transforms, writers)
listener.run()
out_lines = []
with open(outfilename, 'r') as f:
for line in f.readlines():
out_lines.append(line.rstrip())
out_lines.sort()
source_lines = []
for f in SAMPLE_DATA:
source_lines.extend(
['prefix_2 prefix_1 ' + f for f in SAMPLE_DATA[f]])
source_lines.sort()
logging.debug('out: %s, source: %s', out_lines, source_lines)
self.assertEqual(out_lines, source_lines)
def test_basic(self):
# Create a file
temp_dir = tempfile.TemporaryDirectory()
temp_dir_name = temp_dir.name
test_file = temp_dir_name + '/test.txt'
logging.info('creating temporary file "%s"', test_file)
client_writer = TextFileWriter(filename=test_file)
timeout_writer = TimeoutWriter(writer=client_writer,
timeout=0.5,
message='off',
resume_message='on')
time.sleep(0.75) # trigger an "off" message
timeout_writer.write('foo') # trigger an "on" message
time.sleep(1.2) # should trigger just one "off" message
timeout_writer.quit()
# Now check the file - make sure we got
with open(test_file, 'r') as f:
lines = f.read().strip().split('\n')
self.assertEqual(lines[0], 'off')
self.assertEqual(lines[1], 'on')
self.assertEqual(lines[2], 'off')
self.assertEqual(len(lines), 3)
def setUp(self):
# To suppress resource warnings about unclosed files
warnings.simplefilter("ignore", ResourceWarning)
# Create a file
self.temp_dir = tempfile.TemporaryDirectory()
self.temp_dir_name = self.temp_dir.name
self.source_name = self.temp_dir_name + '/source.txt'
self.dest_name = self.temp_dir_name + '/dest.txt'
# Create the source file
writer = TextFileWriter(self.source_name)
for line in SAMPLE_DATA:
writer.write(line)
self.config = CONFIG
self.config['readers']['kwargs']['file_spec'] = self.source_name
self.config['writers']['kwargs']['filename'] = self.dest_name
def test_check_format(self):
f1_name = self.tmpdirname + '/f1'
f2_name = self.tmpdirname + '/f2'
# This should be okay
ComposedWriter(
transforms=[PrefixTransform('prefix')],
writers=[TextFileWriter(f1_name),
TextFileWriter(f2_name)],
check_format=True)
# Should raise an error if formats are not compatible
with self.assertLogs(logging.getLogger(), logging.ERROR):
with self.assertRaises(ValueError):
ComposedWriter(
transforms=[ParseNMEATransform()],
writers=[TextFileWriter(f1_name),
TextFileWriter(f2_name)],
check_format=True)
def test_formats(self):
writer = TextFileWriter(filename=None)
self.assertEqual(writer.input_format(), formats.Text)
self.assertEqual(writer.input_format(formats.NMEA), formats.NMEA)
self.assertEqual(writer.input_format(), formats.NMEA)
with self.assertRaises(TypeError):
writer.input_format('not a format')
def write(self, record):
"""Note: Assume record begins with a timestamp string."""
if record is None:
return
# If we've got a list, hope it's a list of records. Recurse,
# calling write() on each of the list elements in order.
if type(record) is list:
for single_record in record:
self.write(single_record)
return
if not type(record) is str:
logging.error(
'LogfileWriter.write() - record is not timestamped string: '
'%s', record)
return
# First things first: get the date string from the record
try:
time_str = record.split(self.split_char)[0]
ts = timestamp.timestamp(time_str, time_format=self.time_format)
hr_str = self.rollover_hourly and timestamp.date_str(
ts, date_format='_%H00') or ""
date_str = timestamp.date_str(ts, date_format=self.date_format)
logging.debug('LogfileWriter date_str: %s', date_str)
except ValueError:
logging.error('LogfileWriter.write() - bad record timestamp: %s',
record)
return
# Is it time to create a new file to write to?
if not self.writer or date_str != self.current_date or hr_str != self.current_hour:
self.current_filename = self.filebase + '-' + date_str + hr_str + self.suffix
self.current_date = date_str
self.current_hour = self.rollover_hourly and hr_str or ""
logging.info('LogfileWriter opening new file: %s',
self.current_filename)
self.writer = TextFileWriter(self.current_filename, self.flush)
logging.debug('LogfileWriter writing record: %s', record)
self.writer.write(record)
def run(self, loop=False):
"""Create the virtual port with socat and start feeding it records from
the designated logfile. If loop==True, loop when reaching end of input."""
self.socat_thread = threading.Thread(target=self._run_socat,
daemon=True)
self.socat_thread.start()
time.sleep(0.2)
self.reader = LogfileReader(filebase=self.source_file,
use_timestamps=self.use_timestamps,
time_format=self.time_format)
self.strip = SliceTransform('1:') # strip off the first field)
self.writer = TextFileWriter(self.write_port, truncate=True)
while not self.quit:
try:
record = self.reader.read() # get the next record
logging.debug('SimSerial got: %s', record)
# End of input? If loop==True, re-open the logfile from the start
if record is None:
if not loop:
break
self.reader = LogfileReader(
filebase=self.source_file,
use_timestamps=self.use_timestamps)
record = self.strip.transform(record) # strip the timestamp
if record:
logging.debug('SimSerial writing: %s', record)
self.writer.write(
record) # and write it to the virtual port
except (OSError, KeyboardInterrupt):
break
# If we're here, we got None from our input, and are done. Signal
# for run_socat to exit
self.quit = True
def test_all_files(self):
f1_name = self.tmpdirname + '/f1'
f2_name = self.tmpdirname + '/f2'
writer = ComposedWriter(
transforms=[],
writers=[TextFileWriter(f1_name),
TextFileWriter(f2_name)])
f1 = open(f1_name, 'r')
f2 = open(f2_name, 'r')
for line in SAMPLE_DATA:
writer.write(line)
time.sleep(0.1)
f1_line = f1.readline().rstrip()
f2_line = f2.readline().rstrip()
logging.info('wrote: "%s", f1: "%s", f2: "%s"', line, f1_line,
f2_line)
self.assertEqual(line, f1_line)
self.assertEqual(line, f2_line)
def test_read_one_write_all(self):
readers = TextFileReader(self.tmpfilenames[0])
outfilenames = [
self.tmpdirname + '/' + f for f in ['f1_out', 'f2_out', 'f3_out']
]
writers = [TextFileWriter(ofn) for ofn in outfilenames]
listener = Listener(readers=readers, writers=writers)
listener.run()
for ofn in outfilenames:
line_num = 0
with open(ofn, 'r') as f:
for line in f.readlines():
self.assertEqual(SAMPLE_DATA['f1'][line_num],
line.rstrip())
line_num += 1
class SimSerial:
"""Create a virtual serial port and feed stored logfile data to it."""
############################
def __init__(self,
port,
source_file,
use_timestamps=True,
baudrate=9600,
bytesize=8,
parity='N',
stopbits=1,
timeout=None,
xonxoff=False,
rtscts=False,
write_timeout=None,
dsrdtr=False,
inter_byte_timeout=None,
exclusive=None):
"""Takes source file, whether to deliver data at rate indicated by
timestamps, and the standard parameters that a serial port takes."""
self.source_file = source_file
self.use_timestamps = use_timestamps
# We'll create two virtual ports: 'port' and 'port_in'; we will write
# to port_in and read the values back out from port
self.read_port = port
self.write_port = port + '_in'
self.serial_params = {
'baudrate': baudrate,
'byteside': bytesize,
'parity': parity,
'stopbits': stopbits,
'timeout': timeout,
'xonxoff': xonxoff,
'rtscts': rtscts,
'write_timeout': write_timeout,
'dsrdtr': dsrdtr,
'inter_byte_timeout': inter_byte_timeout,
'exclusive': exclusive
}
self.quit = False
# Finally, check that our needed 'socat' actually exists
if not subprocess.run(['which', 'socat'],
stdout=subprocess.PIPE).stdout:
raise NameError(
'Executable "socat" not found on path. Please refer '
'to installation guide to install socat.')
############################
def _run_socat(self):
"""Internal: run the actual command."""
verbose = '-d'
write_port_params = 'pty,link=%s,raw,echo=0' % self.write_port
read_port_params = 'pty,link=%s,raw,echo=0' % self.read_port
cmd = [
'/usr/bin/env',
'socat',
verbose,
#verbose, # repeating makes it more verbose
read_port_params,
write_port_params,
]
try:
# Run socat process using Popen, checking every second or so whether
# it's died (poll() != None) or we've gotten a quit signal.
logging.info('Calling: %s', ' '.join(cmd))
socat_process = subprocess.Popen(cmd)
while not self.quit and not socat_process.poll():
try:
socat_process.wait(1)
except subprocess.TimeoutExpired:
pass
except Exception as e:
logging.error('ERROR: socat command: %s', e)
# If here, process has terminated, or we've seen self.quit. We
# want both to be true: if we've terminated, set self.quit so that
# 'run' loop can exit. If self.quit, terminate process.
if self.quit:
socat_process.kill()
else:
self.quit = True
logging.info('Finished: %s', ' '.join(cmd))
############################
def run(self):
"""Create the virtual port with socat and start feeding it records from
the designated logfile."""
self.socat_thread = threading.Thread(target=self._run_socat)
self.socat_thread.start()
time.sleep(0.2)
self.reader = LogfileReader(filebase=self.source_file,
use_timestamps=self.use_timestamps)
self.strip = SliceTransform('1:') # strip off the first field)
self.writer = TextFileWriter(self.write_port, truncate=True)
while not self.quit:
record = self.reader.read() # get the next record
logging.debug('SimSerial got: %s', record)
if record is None:
break
record = self.strip.transform(record) # strip the timestamp
if record:
logging.debug('SimSerial writing: %s', record)
self.writer.write(record) # and write it to the virtual port
# If we're here, we got None from our input, and are done. Signal
# for run_socat to exit
self.quit = True
class SimSerial:
"""Create a virtual serial port and feed stored logfile data to it."""
############################
def __init__(self, port, prefix=None, timestamp=False,
time_format=TIME_FORMAT, filebase=None, eol='\n',
baudrate=9600, bytesize=8, parity='N', stopbits=1,
timeout=None, xonxoff=False, rtscts=False, write_timeout=None,
dsrdtr=False, inter_byte_timeout=None, exclusive=None):
"""
Simulate a serial port, feeding it data from the specified file.
```
port - Temporary serial port to create and make available for reading
records.
prefix - If non-empty, prefix name prefix to add
timestamp = If True, apply current timestamp to record
time_format - What format to use for timestamp
```
"""
# We'll create two virtual ports: 'port' and 'port_in'; we will write
# to port_in and read the values back out from port
self.read_port = port
self.write_port = port + '_in'
self.prefix = PrefixTransform(prefix) if prefix else None
self.timestamp = TimestampTransform() if timestamp else None
self.time_format = time_format
self.filebase = filebase
self.serial_params = None
# Complain, but go ahead if read_port or write_port exist.
for path in [self.read_port, self.write_port]:
if os.path.exists(path):
logging.warning('Path %s exists; overwriting!')
# Do we have any files we can actually read from?
if not glob.glob(filebase + '*'):
logging.warning('%s: no files matching "%s*"', prefix, filebase)
return
# Set up our parameters
self.serial_params = {'baudrate': baudrate,
'byteside': bytesize,
'parity': parity,
'stopbits': stopbits,
'timeout': timeout,
'xonxoff': xonxoff,
'rtscts': rtscts,
'write_timeout': write_timeout,
'dsrdtr': dsrdtr,
'inter_byte_timeout': inter_byte_timeout,
'exclusive': exclusive}
self.quit = False
# Finally, find path to socat executable
self.socat_path = None
for socat_path in ['/usr/bin/socat', '/usr/local/bin/socat']:
if os.path.exists(socat_path) and os.path.isfile(socat_path):
self.socat_path = socat_path
if not self.socat_path:
raise NameError('Executable "socat" not found on path. Please refer '
'to installation guide to install socat.')
############################
def _run_socat(self):
"""Internal: run the actual command."""
verbose = '-d'
write_port_params = 'pty,link=%s,raw,echo=0' % self.write_port
read_port_params = 'pty,link=%s,raw,echo=0' % self.read_port
cmd = [self.socat_path,
verbose,
#verbose, # repeating makes it more verbose
read_port_params,
write_port_params,
]
try:
# Run socat process using Popen, checking every second or so whether
# it's died (poll() != None) or we've gotten a quit signal.
logging.info('Calling: %s', ' '.join(cmd))
socat_process = subprocess.Popen(cmd)
while not self.quit and not socat_process.poll():
try:
socat_process.wait(1)
except subprocess.TimeoutExpired:
pass
except Exception as e:
logging.error('ERROR: socat command: %s', e)
# If here, process has terminated, or we've seen self.quit. We
# want both to be true: if we've terminated, set self.quit so that
# 'run' loop can exit. If self.quit, terminate process.
if self.quit:
socat_process.kill()
else:
self.quit = True
logging.info('Finished: %s', ' '.join(cmd))
############################
def run(self, loop=False):
# If self.serial_params is None, it means that either read or
# write device already exist, so we shouldn't actually run, or
# we'll destroy them.
if not self.serial_params:
return
"""Create the virtual port with socat and start feeding it records from
the designated logfile. If loop==True, loop when reaching end of input."""
self.socat_thread = threading.Thread(target=self._run_socat, daemon=True)
self.socat_thread.start()
time.sleep(0.2)
self.reader = LogfileReader(filebase=self.filebase, use_timestamps=True,
time_format=self.time_format)
self.slice_n = SliceTransform('1:') # strip off the first field (timestamp)
self.writer = TextFileWriter(self.write_port, truncate=True)
logging.info('Starting %s: %s', self.read_port, self.filebase)
while not self.quit:
try:
record = self.reader.read() # get the next record
logging.debug('SimSerial got: %s', record)
# End of input? If loop==True, re-open the logfile from the start
if record is None:
if not loop:
break
self.reader = LogfileReader(filebase=self.filebase,
use_timestamps=True,
time_format=self.time_format)
record = self.slice_n.transform(record) # strip the timestamp
if not record:
continue
if self.timestamp: # do we want to add a timestamp?
record = self.timestamp.transform(record)
if self.prefix: # do we want to add prefix?
record = self.prefix.transform(record)
logging.debug('SimSerial writing: %s', record)
self.writer.write(record) # and write it to the virtual port
except (OSError, KeyboardInterrupt):
break
# If we're here, we got None from our input, and are done. Signal
# for run_socat to exit
self.quit = True
def test_compatible(self):
# Don't specify 'tail' and expect there to be no data
with tempfile.TemporaryDirectory() as tmpdirname:
writer = TextFileWriter(tmpdirname + '/f')
reader = TextFileReader(tmpdirname + '/f')
self.assertTrue(writer.can_accept(reader))
transforms.append(FromJSONTransform())
if new_args.from_json_to_das_record:
transforms.append(FromJSONTransform(das_record=True))
if new_args.to_das_record:
transforms.append(
ToDASRecordTransform(field_name=new_args.to_das_record))
##########################
# Writers
if new_args.write_file:
for filename in new_args.write_file.split(','):
if filename == '-':
filename = None
writers.append(TextFileWriter(filename=filename))
if new_args.write_logfile:
writers.append(LogfileWriter(filebase=new_args.write_logfile))
if new_args.write_network:
eol = all_args.network_eol
for addr in new_args.write_network.split(','):
writers.append(NetworkWriter(network=addr, eol=eol))
if new_args.write_udp:
eol = all_args.network_eol
for addr_str in new_args.write_udp.split(','):
addr = addr_str.split(':')
dest = ''
interface = ''
dest='verbosity',
default=0,
action='count',
help='Increase output verbosity')
args = parser.parse_args()
# Set logging verbosity
LOGGING_FORMAT = '%(asctime)-15s %(filename)s:%(lineno)d %(message)s'
LOG_LEVELS = {0: logging.WARNING, 1: logging.INFO, 2: logging.DEBUG}
log_level = LOG_LEVELS[min(args.verbosity, max(LOG_LEVELS))]
logging.basicConfig(format=LOGGING_FORMAT)
logging.getLogger().setLevel(log_level)
if args.stderr_file:
stderr_writer = [
TextFileWriter(filename=args.stderr_file, split_by_date=True)
]
logging.getLogger().addHandler(StdErrLoggingHandler(stderr_writer))
logging.info('Starting CachedDataServer')
server = CachedDataServer(port=args.port,
interval=args.interval,
back_seconds=args.back_seconds,
max_records=args.max_records,
cleanup_interval=args.cleanup_interval,
disk_cache=args.disk_cache)
# Only create reader(s) if they've given us a network to read from;
# otherwise, count on data coming from websocket publish
# connections.
if args.udp:
class LogfileWriter(Writer):
"""Write to the specified file. If filename is empty, write to stdout."""
def __init__(self,
filebase=None,
flush=True,
time_format=timestamp.TIME_FORMAT,
date_format=timestamp.DATE_FORMAT,
split_char=' ',
suffix='',
rollover_hourly=False):
"""
Write timestamped text records to file. Base filename will have
date appended, in keeping with R2R format recommendations
(http://www.rvdata.us/operators/directory). When timestamped date on
records rolls over to next day, create new file with new date suffix.
```
filebase Base name of file to write to. Will have record date
appended, e.g.
flush If True (default), flush after every write() call
date_fomat A strftime-compatible string, such as '%Y-%m-%d';
defaults to whatever's defined in
utils.timestamps.DATE_FORMAT.
split_char Delimiter between timestamp and rest of message
suffix string to apply to the end of the log filename
rollover_hourly Set files to truncate by hour. By default files will
truncate by day
```
"""
super().__init__(input_format=Text)
self.filebase = filebase
self.flush = flush
self.time_format = time_format
self.date_format = date_format
self.split_char = split_char
self.suffix = suffix
self.rollover_hourly = rollover_hourly
self.current_date = None
self.current_hour = None
self.current_filename = None
self.writer = None
############################
def write(self, record):
"""Note: Assume record begins with a timestamp string."""
if record is None:
return
# If we've got a list, hope it's a list of records. Recurse,
# calling write() on each of the list elements in order.
if type(record) is list:
for single_record in record:
self.write(single_record)
return
if not type(record) is str:
logging.error(
'LogfileWriter.write() - record is not timestamped string: '
'%s', record)
return
# First things first: get the date string from the record
try:
time_str = record.split(self.split_char)[0]
ts = timestamp.timestamp(time_str, time_format=self.time_format)
hr_str = self.rollover_hourly and timestamp.date_str(
ts, date_format='_%H00') or ""
date_str = timestamp.date_str(ts, date_format=self.date_format)
logging.debug('LogfileWriter date_str: %s', date_str)
except ValueError:
logging.error('LogfileWriter.write() - bad record timestamp: %s',
record)
return
# Is it time to create a new file to write to?
if not self.writer or date_str != self.current_date or hr_str != self.current_hour:
self.current_filename = self.filebase + '-' + date_str + hr_str + self.suffix
self.current_date = date_str
self.current_hour = self.rollover_hourly and hr_str or ""
logging.info('LogfileWriter opening new file: %s',
self.current_filename)
self.writer = TextFileWriter(self.current_filename, self.flush)
logging.debug('LogfileWriter writing record: %s', record)
self.writer.write(record)
'--verbosity',
dest='verbosity',
default=0,
action='count',
help='Increase output verbosity')
args = parser.parse_args()
LOGGING_FORMAT = '%(asctime)-15s %(message)s'
logging.basicConfig(format=LOGGING_FORMAT)
LOG_LEVELS = {0: logging.WARNING, 1: logging.INFO, 2: logging.DEBUG}
args.verbosity = min(args.verbosity, max(LOG_LEVELS))
logging.getLogger().setLevel(LOG_LEVELS[args.verbosity])
reader = TextFileReader(file_spec=args.read, tail=args.tail)
writer = TextFileWriter(filename=args.write)
if args.prefix:
prefix_transform = PrefixTransform(args.prefix)
if args.timestamp:
timestamp_transform = TimestampTransform()
while True:
record = reader.read()
now = time.time()
logging.info('Got record: %s', record)
if record is None:
if not args.tail:
action='count',
help='Increase output verbosity')
parser.add_argument('-V',
'--logger_verbosity',
dest='logger_verbosity',
default=0,
action='count',
help='Increase output verbosity of component loggers')
args = parser.parse_args()
# Set up logging first of all
LOG_LEVELS = {0: logging.WARNING, 1: logging.INFO, 2: logging.DEBUG}
log_level = LOG_LEVELS[min(args.verbosity, max(LOG_LEVELS))]
if args.stderr_file:
stderr_writers = [TextFileWriter(args.stderr_file)]
logging.getLogger().addHandler(StdErrLoggingHandler(stderr_writers))
# What level do we want our component loggers to write?
logger_log_level = LOG_LEVELS[min(args.logger_verbosity, max(LOG_LEVELS))]
configs = read_config(args.config) if args.config else None
# If they've specified a mode, see if what we've read in is in fact
# a cruise definition with modes we can choose from.
if args.mode:
modes = configs.get('modes', None)
if not modes:
logging.fatal('--mode specified, but "%s" has no modes' %
args.config)
sys.exit(1)