def test_write_no_delimiter(self):
with tempfile.TemporaryDirectory() as tmpdirname:
writer = FileWriter(tmpdirname + '/f', delimiter=None)
for line in SAMPLE_DATA:
writer.write(line)
with open(tmpdirname + '/f') as f:
self.assertEqual(f.readline(), ''.join(SAMPLE_DATA))
def test_write(self):
with tempfile.TemporaryDirectory() as tmpdirname:
writer = FileWriter(tmpdirname + '/f')
for line in SAMPLE_DATA:
writer.write(line)
with open(tmpdirname + '/f') as f:
for line in SAMPLE_DATA:
self.assertEqual(line, f.readline().strip())
def write(self, record):
"""Note: Assume record begins with a timestamp string."""
if record is None:
return
# If we've got a list, hope it's a list of records. Recurse,
# calling write() on each of the list elements in order.
if type(record) is list:
for single_record in record:
self.write(single_record)
return
if not type(record) is str:
logging.error(
'LogfileWriter.write() - record not timestamped: %s ', record)
return
# Get the timestamp we'll be using
try: # Try to extract timestamp from record
time_str = record.split(self.split_char)[0]
ts = timestamp.timestamp(time_str, time_format=self.time_format)
except ValueError:
logging.error('LogfileWriter.write() - bad timestamp: %s', record)
return
# Now parse ts into hour and date strings
hr_str = self.rollover_hourly and \
timestamp.date_str(ts, date_format='_%H00') or ""
date_str = timestamp.date_str(ts, date_format=self.date_format)
logging.debug('LogfileWriter date_str: %s', date_str)
# Is it time to create a new file to write to?
if not self.writer or date_str != self.current_date or hr_str != self.current_hour:
self.current_filename = self.filebase + '-' + date_str + hr_str + self.suffix
self.current_date = date_str
self.current_hour = self.rollover_hourly and hr_str or ""
logging.info('LogfileWriter opening new file: %s',
self.current_filename)
self.writer = FileWriter(filename=self.current_filename,
flush=self.flush)
logging.debug('LogfileWriter writing record: %s', record)
self.writer.write(record)
def test_split_day(self):
"""Test the split_by_date parameter, changing the date with each write."""
with tempfile.TemporaryDirectory() as tmpdirname:
writer = FileWriter(tmpdirname + '/g', split_by_time=True)
writer.timestamp = 1597150898
writer.write(SAMPLE_DATA[0])
writer.timestamp += 86400
writer.write(SAMPLE_DATA[1])
writer.timestamp += 86400
writer.write(SAMPLE_DATA[2])
with open(tmpdirname + '/g-2020-08-11') as f:
self.assertEqual(SAMPLE_DATA[0], f.readline().strip())
with open(tmpdirname + '/g-2020-08-12') as f:
self.assertEqual(SAMPLE_DATA[1], f.readline().strip())
with open(tmpdirname + '/g-2020-08-13') as f:
self.assertEqual(SAMPLE_DATA[2], f.readline().strip())
transforms.append(FromJSONTransform())
if new_args.from_json_to_das_record:
transforms.append(FromJSONTransform(das_record=True))
if new_args.to_das_record:
transforms.append(
ToDASRecordTransform(field_name=new_args.to_das_record))
##########################
# Writers
if new_args.write_file:
for filename in new_args.write_file.split(','):
if filename == '-':
filename = None
writers.append(FileWriter(filename=filename))
if new_args.write_logfile:
writers.append(LogfileWriter(filebase=new_args.write_logfile))
if new_args.write_network:
eol = all_args.network_eol
for addr in new_args.write_network.split(','):
writers.append(NetworkWriter(network=addr, eol=eol))
if new_args.write_udp:
eol = all_args.network_eol
for addr_str in new_args.write_udp.split(','):
addr = addr_str.split(':')
dest = ''
interface = ''