def write(self, msg): if self.current_file_records_count[0] >= self.settings.get_max_records_per_file(): try: self.current_file = self.create_datafile() log.debug("FileStorage_writer -- Created new data file: %s", self.current_file) except IOError as e: log.error("Failed to create a new file! %s", e) self.files.get_data_files().append(self.current_file) self.current_file_records_count[0] = 0 try: if self.buffered_writer is not None and self.buffered_writer.closed is False: self.buffered_writer.close() except IOError as e: log.warning("Failed to close buffered writer! %s", e) self.buffered_writer = None try: encoded = b64encode(msg.encode("utf-8")) self.buffered_writer = self.get_or_init_buffered_writer(self.current_file) self.buffered_writer.write(encoded + linesep.encode('utf-8')) # self.buffered_writer.write(linesep.encode('utf-8')) self.current_file_records_count[0] += 1 if self.current_file_records_count[0] - self.previous_file_records_count[0] >= self.settings.get_max_records_between_fsync(): self.previous_file_records_count = self.current_file_records_count[:] self.buffered_writer.flush() try: if self.buffered_writer is not None and self.buffered_writer.closed is False: self.buffered_writer.close() except IOError as e: log.warning("Failed to close buffered writer! %s", e) except IOError as e: log.warning("Failed to update data file![%s]\n%s", self.current_file, e)
def read_state_file(self): try: state_data_node = {} try: with BufferedReader(FileIO(self.settings.get_data_folder_path() + self.files.get_state_file(), 'r')) as br: state_data_node = load(br) except JSONDecodeError: log.error("Failed to decode JSON from state file") state_data_node = 0 except IOError as e: log.warning("Failed to fetch info from state file!", e) reader_file = None reader_pos = 0 if state_data_node: reader_pos = state_data_node['position'] for file in sorted(self.files.get_data_files()): if file == state_data_node['file']: reader_file = file break if reader_file is None: reader_file = sorted(self.files.get_data_files())[0] reader_pos = 0 log.info("FileStorage_reader -- Initializing from state file: [%s:%i]", self.settings.get_data_folder_path() + reader_file, reader_pos) return EventStorageReaderPointer(reader_file, reader_pos) except Exception as e: log.exception(e)
def get_or_init_buffered_reader(self, pointer): try: if self.buffered_reader is None or self.buffered_reader.closed: new_file_to_read_path = self.settings.get_data_folder_path( ) + pointer.get_file() # if not exists(new_file_to_read_path): # next_file = self.get_next_file(self.files, self.new_pos) # if next_file is not None: # new_file_to_read_path = self.settings.get_data_folder_path() + next_file # else: # self.buffered_reader = None # return None self.buffered_reader = BufferedReader( FileIO(new_file_to_read_path, 'r')) lines_to_skip = pointer.get_line() if lines_to_skip > 0: while self.buffered_reader.readline() is not None: if lines_to_skip > 0: lines_to_skip -= 1 else: break return self.buffered_reader except IOError as e: log.error("Failed to initialize buffered reader! Error: %s", e) raise RuntimeError("Failed to initialize buffered reader!", e) except Exception as e: log.exception(e)
def create_file(self, prefix, filename): full_file_name = "%s%s.txt" % (prefix, filename) file_path = "%s%s" % (self.settings.get_data_folder_path(), full_file_name) try: os_open(file_path, O_CREAT | O_EXCL) return full_file_name except IOError as e: log.error("Failed to create a new file! Error: %s", e)
def get_or_init_buffered_writer(self, file): try: if self.buffered_writer is None or self.buffered_writer.closed: self.buffered_writer = BufferedWriter(FileIO(self.settings.get_data_folder_path() + file, 'a')) return self.buffered_writer except IOError as e: log.error("Failed to initialize buffered writer! Error: %s", e) raise RuntimeError("Failed to initialize buffered writer!", e)
def write(self, msg): if len(self.files.data_files) <= self.settings.get_max_files_count(): if self.current_file_records_count[ 0] >= self.settings.get_max_records_per_file( ) or not exists(self.settings.get_data_folder_path() + self.current_file): try: self.current_file = self.create_datafile() log.debug( "FileStorage_writer -- Created new data file: %s", self.current_file) except IOError as e: log.error("Failed to create a new file! %s", e) self.files.get_data_files().append(self.current_file) self.current_file_records_count[0] = 0 try: if self.buffered_writer is not None and self.buffered_writer.closed is False: self.buffered_writer.close() except IOError as e: log.warning("Failed to close buffered writer! %s", e) self.buffered_writer = None try: encoded = b64encode(msg.encode("utf-8")) if not exists(self.settings.get_data_folder_path() + self.current_file): self.current_file = self.create_datafile() self.buffered_writer = self.get_or_init_buffered_writer( self.current_file) self.buffered_writer.write(encoded + linesep.encode('utf-8')) self.current_file_records_count[0] += 1 if self.current_file_records_count[ 0] - self.previous_file_records_count[ 0] >= self.settings.get_max_records_between_fsync( ): self.previous_file_records_count = self.current_file_records_count[:] self.buffered_writer.flush() try: if self.buffered_writer is not None and self.buffered_writer.closed is False: self.buffered_writer.close() except IOError as e: log.warning("Failed to close buffered writer! %s", e) except IOError as e: log.warning("Failed to update data file![%s]\n%s", self.current_file, e) else: raise DataFileCountError( "The number of data files has been exceeded - change the settings or check the connection. New data will be lost." )
def get_or_init_buffered_reader(self, pointer): try: if self.buffered_reader is None or self.buffered_reader.closed: new_file_to_read_path = self.settings.get_data_folder_path() + pointer.get_file() self.buffered_reader = BufferedReader(FileIO(new_file_to_read_path, 'r')) lines_to_skip = pointer.get_line() if lines_to_skip > 0: while self.buffered_reader.readline() is not None: if lines_to_skip != 0: lines_to_skip -= 1 else: break return self.buffered_reader except IOError as e: log.error("Failed to initialize buffered reader!", e) raise RuntimeError("Failed to initialize buffered reader!", e)