def test_write_no_delimiter(self): with tempfile.TemporaryDirectory() as tmpdirname: writer = FileWriter(tmpdirname + '/f', delimiter=None) for line in SAMPLE_DATA: writer.write(line) with open(tmpdirname + '/f') as f: self.assertEqual(f.readline(), ''.join(SAMPLE_DATA))
def test_write(self): with tempfile.TemporaryDirectory() as tmpdirname: writer = FileWriter(tmpdirname + '/f') for line in SAMPLE_DATA: writer.write(line) with open(tmpdirname + '/f') as f: for line in SAMPLE_DATA: self.assertEqual(line, f.readline().strip())
def write(self, record): """Note: Assume record begins with a timestamp string.""" if record is None: return # If we've got a list, hope it's a list of records. Recurse, # calling write() on each of the list elements in order. if type(record) is list: for single_record in record: self.write(single_record) return if not type(record) is str: logging.error( 'LogfileWriter.write() - record not timestamped: %s ', record) return # Get the timestamp we'll be using try: # Try to extract timestamp from record time_str = record.split(self.split_char)[0] ts = timestamp.timestamp(time_str, time_format=self.time_format) except ValueError: logging.error('LogfileWriter.write() - bad timestamp: %s', record) return # Now parse ts into hour and date strings hr_str = self.rollover_hourly and \ timestamp.date_str(ts, date_format='_%H00') or "" date_str = timestamp.date_str(ts, date_format=self.date_format) logging.debug('LogfileWriter date_str: %s', date_str) # Is it time to create a new file to write to? if not self.writer or date_str != self.current_date or hr_str != self.current_hour: self.current_filename = self.filebase + '-' + date_str + hr_str + self.suffix self.current_date = date_str self.current_hour = self.rollover_hourly and hr_str or "" logging.info('LogfileWriter opening new file: %s', self.current_filename) self.writer = FileWriter(filename=self.current_filename, flush=self.flush) logging.debug('LogfileWriter writing record: %s', record) self.writer.write(record)
def test_split_day(self): """Test the split_by_date parameter, changing the date with each write.""" with tempfile.TemporaryDirectory() as tmpdirname: writer = FileWriter(tmpdirname + '/g', split_by_time=True) writer.timestamp = 1597150898 writer.write(SAMPLE_DATA[0]) writer.timestamp += 86400 writer.write(SAMPLE_DATA[1]) writer.timestamp += 86400 writer.write(SAMPLE_DATA[2]) with open(tmpdirname + '/g-2020-08-11') as f: self.assertEqual(SAMPLE_DATA[0], f.readline().strip()) with open(tmpdirname + '/g-2020-08-12') as f: self.assertEqual(SAMPLE_DATA[1], f.readline().strip()) with open(tmpdirname + '/g-2020-08-13') as f: self.assertEqual(SAMPLE_DATA[2], f.readline().strip())
transforms.append(FromJSONTransform()) if new_args.from_json_to_das_record: transforms.append(FromJSONTransform(das_record=True)) if new_args.to_das_record: transforms.append( ToDASRecordTransform(field_name=new_args.to_das_record)) ########################## # Writers if new_args.write_file: for filename in new_args.write_file.split(','): if filename == '-': filename = None writers.append(FileWriter(filename=filename)) if new_args.write_logfile: writers.append(LogfileWriter(filebase=new_args.write_logfile)) if new_args.write_network: eol = all_args.network_eol for addr in new_args.write_network.split(','): writers.append(NetworkWriter(network=addr, eol=eol)) if new_args.write_udp: eol = all_args.network_eol for addr_str in new_args.write_udp.split(','): addr = addr_str.split(':') dest = '' interface = ''