def read_state_file(self): try: state_data_node = {} try: with BufferedReader( FileIO( self.settings.get_data_folder_path() + self.files.get_state_file(), 'r')) as br: state_data_node = load(br) except JSONDecodeError: log.error("Failed to decode JSON from state file") state_data_node = 0 except IOError as e: log.warning("Failed to fetch info from state file!", e) reader_file = None reader_pos = 0 if state_data_node: reader_pos = state_data_node['position'] for file in sorted(self.files.get_data_files()): if file == state_data_node['file']: reader_file = file break if reader_file is None: reader_file = sorted(self.files.get_data_files())[0] reader_pos = 0 log.info( "FileStorage_reader -- Initializing from state file: [%s:%i]", self.settings.get_data_folder_path() + reader_file, reader_pos) return EventStorageReaderPointer(reader_file, reader_pos) except Exception as e: log.exception(e)
def read(self): if self.current_batch is not None and self.current_batch: log.debug("The previous batch was not discarded!") return self.current_batch self.current_batch = [] records_to_read = self.settings.get_max_read_records_count() while records_to_read > 0: try: current_line_in_file = self.new_pos.get_line() self.buffered_reader = self.get_or_init_buffered_reader(self.new_pos) line = self.buffered_reader.readline() while line != b'': try: self.current_batch.append(b64decode(line).decode("utf-8")) records_to_read -= 1 except IOError as e: log.warning("Could not parse line [%s] to uplink message! %s", line, e) except Exception as e: log.exception(e) current_line_in_file += 1 self.new_pos.set_line(current_line_in_file) self.write_info_to_state_file(self.new_pos) break finally: current_line_in_file += 1 if records_to_read > 0: line = self.buffered_reader.readline() self.new_pos.set_line(current_line_in_file) if records_to_read == 0: break if current_line_in_file >= self.settings.get_max_records_per_file(): previous_file = self.new_pos next_file = self.get_next_file(self.files, self.new_pos) if next_file is not None: if self.buffered_reader is not None: self.buffered_reader.close() self.buffered_reader = None self.delete_read_file(previous_file) self.new_pos = EventStorageReaderPointer(next_file, 0) self.write_info_to_state_file(self.new_pos) continue else: # No more records to read for now break # continue ################### if line == b'': break ####################### else: # No more records to read for now continue except IOError as e: log.warning("[{}] Failed to read file!".format(self.new_pos.get_file(), e)) break except Exception as e: log.exception(e) return self.current_batch