def read_state_file(self):
try:
state_data_node = {}
try:
with BufferedReader(FileIO(self.settings.get_data_folder_path() +
self.files.get_state_file(), 'r')) as br:
state_data_node = load(br)
except JSONDecodeError:
log.error("Failed to decode JSON from state file")
state_data_node = 0
except IOError as e:
log.warning("Failed to fetch info from state file!", e)
reader_file = None
reader_pos = 0
if state_data_node:
reader_pos = state_data_node['position']
for file in sorted(self.files.get_data_files()):
if file == state_data_node['file']:
reader_file = file
break
if reader_file is None:
reader_file = sorted(self.files.get_data_files())[0]
reader_pos = 0
log.info("FileStorage_reader -- Initializing from state file: [%s:%i]",
self.settings.get_data_folder_path() + reader_file,
reader_pos)
return EventStorageReaderPointer(reader_file, reader_pos)
except Exception as e:
log.exception(e)
def write(self, msg):
if self.current_file_records_count[0] >= self.settings.get_max_records_per_file():
try:
self.current_file = self.create_datafile()
log.debug("FileStorage_writer -- Created new data file: %s", self.current_file)
except IOError as e:
log.error("Failed to create a new file! %s", e)
self.files.get_data_files().append(self.current_file)
self.current_file_records_count[0] = 0
try:
if self.buffered_writer is not None and self.buffered_writer.closed is False:
self.buffered_writer.close()
except IOError as e:
log.warning("Failed to close buffered writer! %s", e)
self.buffered_writer = None
try:
encoded = b64encode(msg.encode("utf-8"))
self.buffered_writer = self.get_or_init_buffered_writer(self.current_file)
self.buffered_writer.write(encoded + linesep.encode('utf-8'))
# self.buffered_writer.write(linesep.encode('utf-8'))
self.current_file_records_count[0] += 1
if self.current_file_records_count[0] - self.previous_file_records_count[0] >= self.settings.get_max_records_between_fsync():
self.previous_file_records_count = self.current_file_records_count[:]
self.buffered_writer.flush()
except IOError as e:
log.warning("Failed to update data file![%s]\n%s", self.current_file, e)
def read(self):
if self.current_batch is not None and self.current_batch:
log.debug("The previous batch was not discarded!")
return self.current_batch
self.current_batch = []
records_to_read = self.settings.get_max_read_records_count()
while records_to_read > 0:
try:
current_line_in_file = self.new_pos.get_line()
self.buffered_reader = self.get_or_init_buffered_reader(self.new_pos)
line = self.buffered_reader.readline()
while line != b'':
try:
self.current_batch.append(b64decode(line).decode("utf-8"))
records_to_read -= 1
except IOError as e:
log.warning("Could not parse line [%s] to uplink message! %s", line, e)
except Exception as e:
log.exception(e)
current_line_in_file += 1
self.new_pos.set_line(current_line_in_file)
self.write_info_to_state_file(self.new_pos)
break
finally:
current_line_in_file += 1
if records_to_read > 0:
line = self.buffered_reader.readline()
self.new_pos.set_line(current_line_in_file)
if records_to_read == 0:
break
if current_line_in_file >= self.settings.get_max_records_per_file():
previous_file = self.new_pos
next_file = self.get_next_file(self.files, self.new_pos)
if next_file is not None:
if self.buffered_reader is not None:
self.buffered_reader.close()
self.buffered_reader = None
self.delete_read_file(previous_file)
self.new_pos = EventStorageReaderPointer(next_file, 0)
self.write_info_to_state_file(self.new_pos)
continue
else:
# No more records to read for now
break
# continue
###################
if line == b'':
break
#######################
else:
# No more records to read for now
continue
except IOError as e:
log.warning("[{}] Failed to read file!".format(self.new_pos.get_file(), e))
break
except Exception as e:
log.exception(e)
return self.current_batch
def write_info_to_state_file(self, pointer: EventStorageReaderPointer):
try:
state_file_node = {'file': pointer.get_file(), 'position': pointer.get_line()}
with open(self.settings.get_data_folder_path() + self.files.get_state_file(), 'w') as outfile:
outfile.write(dumps(state_file_node))
except IOError as e:
log.warning("Failed to update state file!", e)
except Exception as e:
log.exception(e)
def get_number_of_records_in_file(self, file):
if self.current_file_records_count[0] <= 0:
try:
with open(self.settings.get_data_folder_path() + file) as data_file:
for i, _ in enumerate(data_file):
self.current_file_records_count[0] = i + 1
except IOError as e:
log.warning("Could not get the records count from the file![%s] with error: %s", file, e)
except Exception as e:
log.exception(e)
return self.current_file_records_count
def write(self, msg):
if len(self.files.data_files) <= self.settings.get_max_files_count():
if self.current_file_records_count[
0] >= self.settings.get_max_records_per_file(
) or not exists(self.settings.get_data_folder_path() +
self.current_file):
try:
self.current_file = self.create_datafile()
log.debug(
"FileStorage_writer -- Created new data file: %s",
self.current_file)
except IOError as e:
log.error("Failed to create a new file! %s", e)
self.files.get_data_files().append(self.current_file)
self.current_file_records_count[0] = 0
try:
if self.buffered_writer is not None and self.buffered_writer.closed is False:
self.buffered_writer.close()
except IOError as e:
log.warning("Failed to close buffered writer! %s", e)
self.buffered_writer = None
try:
encoded = b64encode(msg.encode("utf-8"))
if not exists(self.settings.get_data_folder_path() +
self.current_file):
self.current_file = self.create_datafile()
self.buffered_writer = self.get_or_init_buffered_writer(
self.current_file)
self.buffered_writer.write(encoded + linesep.encode('utf-8'))
self.current_file_records_count[0] += 1
if self.current_file_records_count[
0] - self.previous_file_records_count[
0] >= self.settings.get_max_records_between_fsync(
):
self.previous_file_records_count = self.current_file_records_count[:]
self.buffered_writer.flush()
try:
if self.buffered_writer is not None and self.buffered_writer.closed is False:
self.buffered_writer.close()
except IOError as e:
log.warning("Failed to close buffered writer! %s", e)
except IOError as e:
log.warning("Failed to update data file![%s]\n%s",
self.current_file, e)
else:
raise DataFileCountError(
"The number of data files has been exceeded - change the settings or check the connection. New data will be lost."
)
