def read_state_file(self): try: state_data_node = {} try: with BufferedReader(FileIO(self.settings.get_data_folder_path() + self.files.get_state_file(), 'r')) as br: state_data_node = load(br) except JSONDecodeError: log.error("Failed to decode JSON from state file") state_data_node = 0 except IOError as e: log.warning("Failed to fetch info from state file!", e) reader_file = None reader_pos = 0 if state_data_node: reader_pos = state_data_node['position'] for file in sorted(self.files.get_data_files()): if file == state_data_node['file']: reader_file = file break if reader_file is None: reader_file = sorted(self.files.get_data_files())[0] reader_pos = 0 log.info("FileStorage_reader -- Initializing from state file: [%s:%i]", self.settings.get_data_folder_path() + reader_file, reader_pos) return EventStorageReaderPointer(reader_file, reader_pos) except Exception as e: log.exception(e)
def write(self, msg): if self.current_file_records_count[0] >= self.settings.get_max_records_per_file(): try: self.current_file = self.create_datafile() log.debug("FileStorage_writer -- Created new data file: %s", self.current_file) except IOError as e: log.error("Failed to create a new file! %s", e) self.files.get_data_files().append(self.current_file) self.current_file_records_count[0] = 0 try: if self.buffered_writer is not None and self.buffered_writer.closed is False: self.buffered_writer.close() except IOError as e: log.warning("Failed to close buffered writer! %s", e) self.buffered_writer = None try: encoded = b64encode(msg.encode("utf-8")) self.buffered_writer = self.get_or_init_buffered_writer(self.current_file) self.buffered_writer.write(encoded + linesep.encode('utf-8')) # self.buffered_writer.write(linesep.encode('utf-8')) self.current_file_records_count[0] += 1 if self.current_file_records_count[0] - self.previous_file_records_count[0] >= self.settings.get_max_records_between_fsync(): self.previous_file_records_count = self.current_file_records_count[:] self.buffered_writer.flush() except IOError as e: log.warning("Failed to update data file![%s]\n%s", self.current_file, e)
def read(self): if self.current_batch is not None and self.current_batch: log.debug("The previous batch was not discarded!") return self.current_batch self.current_batch = [] records_to_read = self.settings.get_max_read_records_count() while records_to_read > 0: try: current_line_in_file = self.new_pos.get_line() self.buffered_reader = self.get_or_init_buffered_reader(self.new_pos) line = self.buffered_reader.readline() while line != b'': try: self.current_batch.append(b64decode(line).decode("utf-8")) records_to_read -= 1 except IOError as e: log.warning("Could not parse line [%s] to uplink message! %s", line, e) except Exception as e: log.exception(e) current_line_in_file += 1 self.new_pos.set_line(current_line_in_file) self.write_info_to_state_file(self.new_pos) break finally: current_line_in_file += 1 if records_to_read > 0: line = self.buffered_reader.readline() self.new_pos.set_line(current_line_in_file) if records_to_read == 0: break if current_line_in_file >= self.settings.get_max_records_per_file(): previous_file = self.new_pos next_file = self.get_next_file(self.files, self.new_pos) if next_file is not None: if self.buffered_reader is not None: self.buffered_reader.close() self.buffered_reader = None self.delete_read_file(previous_file) self.new_pos = EventStorageReaderPointer(next_file, 0) self.write_info_to_state_file(self.new_pos) continue else: # No more records to read for now break # continue ################### if line == b'': break ####################### else: # No more records to read for now continue except IOError as e: log.warning("[{}] Failed to read file!".format(self.new_pos.get_file(), e)) break except Exception as e: log.exception(e) return self.current_batch
def write_info_to_state_file(self, pointer: EventStorageReaderPointer): try: state_file_node = {'file': pointer.get_file(), 'position': pointer.get_line()} with open(self.settings.get_data_folder_path() + self.files.get_state_file(), 'w') as outfile: outfile.write(dumps(state_file_node)) except IOError as e: log.warning("Failed to update state file!", e) except Exception as e: log.exception(e)
def get_number_of_records_in_file(self, file): if self.current_file_records_count[0] <= 0: try: with open(self.settings.get_data_folder_path() + file) as data_file: for i, _ in enumerate(data_file): self.current_file_records_count[0] = i + 1 except IOError as e: log.warning("Could not get the records count from the file![%s] with error: %s", file, e) except Exception as e: log.exception(e) return self.current_file_records_count
def write(self, msg): if len(self.files.data_files) <= self.settings.get_max_files_count(): if self.current_file_records_count[ 0] >= self.settings.get_max_records_per_file( ) or not exists(self.settings.get_data_folder_path() + self.current_file): try: self.current_file = self.create_datafile() log.debug( "FileStorage_writer -- Created new data file: %s", self.current_file) except IOError as e: log.error("Failed to create a new file! %s", e) self.files.get_data_files().append(self.current_file) self.current_file_records_count[0] = 0 try: if self.buffered_writer is not None and self.buffered_writer.closed is False: self.buffered_writer.close() except IOError as e: log.warning("Failed to close buffered writer! %s", e) self.buffered_writer = None try: encoded = b64encode(msg.encode("utf-8")) if not exists(self.settings.get_data_folder_path() + self.current_file): self.current_file = self.create_datafile() self.buffered_writer = self.get_or_init_buffered_writer( self.current_file) self.buffered_writer.write(encoded + linesep.encode('utf-8')) self.current_file_records_count[0] += 1 if self.current_file_records_count[ 0] - self.previous_file_records_count[ 0] >= self.settings.get_max_records_between_fsync( ): self.previous_file_records_count = self.current_file_records_count[:] self.buffered_writer.flush() try: if self.buffered_writer is not None and self.buffered_writer.closed is False: self.buffered_writer.close() except IOError as e: log.warning("Failed to close buffered writer! %s", e) except IOError as e: log.warning("Failed to update data file![%s]\n%s", self.current_file, e) else: raise DataFileCountError( "The number of data files has been exceeded - change the settings or check the connection. New data will be lost." )