def read_state_file(self): try: state_data_node = {} try: with BufferedReader(FileIO(self.settings.get_data_folder_path() + self.files.get_state_file(), 'r')) as br: state_data_node = load(br) except JSONDecodeError: log.error("Failed to decode JSON from state file") state_data_node = 0 except IOError as e: log.warning("Failed to fetch info from state file!", e) reader_file = None reader_pos = 0 if state_data_node: reader_pos = state_data_node['position'] for file in sorted(self.files.get_data_files()): if file == state_data_node['file']: reader_file = file break if reader_file is None: reader_file = sorted(self.files.get_data_files())[0] reader_pos = 0 log.info("FileStorage_reader -- Initializing from state file: [%s:%i]", self.settings.get_data_folder_path() + reader_file, reader_pos) return EventStorageReaderPointer(reader_file, reader_pos) except Exception as e: log.exception(e)
def get_or_init_buffered_reader(self, pointer): try: if self.buffered_reader is None or self.buffered_reader.closed: new_file_to_read_path = self.settings.get_data_folder_path( ) + pointer.get_file() # if not exists(new_file_to_read_path): # next_file = self.get_next_file(self.files, self.new_pos) # if next_file is not None: # new_file_to_read_path = self.settings.get_data_folder_path() + next_file # else: # self.buffered_reader = None # return None self.buffered_reader = BufferedReader( FileIO(new_file_to_read_path, 'r')) lines_to_skip = pointer.get_line() if lines_to_skip > 0: while self.buffered_reader.readline() is not None: if lines_to_skip > 0: lines_to_skip -= 1 else: break return self.buffered_reader except IOError as e: log.error("Failed to initialize buffered reader! Error: %s", e) raise RuntimeError("Failed to initialize buffered reader!", e) except Exception as e: log.exception(e)
def read(self): if self.current_batch is not None and self.current_batch: log.debug("The previous batch was not discarded!") return self.current_batch self.current_batch = [] records_to_read = self.settings.get_max_read_records_count() while records_to_read > 0: try: current_line_in_file = self.new_pos.get_line() self.buffered_reader = self.get_or_init_buffered_reader(self.new_pos) line = self.buffered_reader.readline() while line != b'': try: self.current_batch.append(b64decode(line).decode("utf-8")) records_to_read -= 1 except IOError as e: log.warning("Could not parse line [%s] to uplink message! %s", line, e) except Exception as e: log.exception(e) current_line_in_file += 1 self.new_pos.set_line(current_line_in_file) self.write_info_to_state_file(self.new_pos) break finally: current_line_in_file += 1 if records_to_read > 0: line = self.buffered_reader.readline() self.new_pos.set_line(current_line_in_file) if records_to_read == 0: break if current_line_in_file >= self.settings.get_max_records_per_file(): previous_file = self.new_pos next_file = self.get_next_file(self.files, self.new_pos) if next_file is not None: if self.buffered_reader is not None: self.buffered_reader.close() self.buffered_reader = None self.delete_read_file(previous_file) self.new_pos = EventStorageReaderPointer(next_file, 0) self.write_info_to_state_file(self.new_pos) continue else: # No more records to read for now break # continue ################### if line == b'': break ####################### else: # No more records to read for now continue except IOError as e: log.warning("[{}] Failed to read file!".format(self.new_pos.get_file(), e)) break except Exception as e: log.exception(e) return self.current_batch
def delete_read_file(self, current_file: EventStorageReaderPointer): data_files = self.files.get_data_files() if exists(self.settings.get_data_folder_path() + current_file.file): remove(self.settings.get_data_folder_path() + current_file.file) try: data_files = data_files[1:] except Exception as e: log.exception(e) log.info("FileStorage_reader -- Cleanup old data file: %s%s!", self.settings.get_data_folder_path(), current_file.file)
def write_info_to_state_file(self, pointer: EventStorageReaderPointer): try: state_file_node = {'file': pointer.get_file(), 'position': pointer.get_line()} with open(self.settings.get_data_folder_path() + self.files.get_state_file(), 'w') as outfile: outfile.write(dumps(state_file_node)) except IOError as e: log.warning("Failed to update state file!", e) except Exception as e: log.exception(e)
def discard_batch(self): try: if self.current_pos.get_line() >= self.settings.get_max_records_per_file()-1: if self.buffered_reader is not None and not self.buffered_reader.closed: self.buffered_reader.close() self.write_info_to_state_file(self.new_pos) self.current_pos = self.new_pos self.current_batch = None except Exception as e: log.exception(e)
def get_number_of_records_in_file(self, file): if self.current_file_records_count[0] <= 0: try: with open(self.settings.get_data_folder_path() + file) as data_file: for i, _ in enumerate(data_file): self.current_file_records_count[0] = i + 1 except IOError as e: log.warning("Could not get the records count from the file![%s] with error: %s", file, e) except Exception as e: log.exception(e) return self.current_file_records_count
def delete_read_file(self, current_file: EventStorageReaderPointer): data_files = self.files.get_data_files() try: if exists(self.settings.get_data_folder_path() + current_file.file) and len(data_files) > 1: remove(self.settings.get_data_folder_path() + current_file.file) if current_file.file in data_files: self.files.data_files.remove(current_file.file) log.info("FileStorage_reader -- Cleanup old data file: %s%s!", self.settings.get_data_folder_path(), current_file.file) except Exception as e: log.exception(e)