def write(self, msg):
if self.current_file_records_count[0] >= self.settings.get_max_records_per_file():
try:
self.current_file = self.create_datafile()
log.debug("FileStorage_writer -- Created new data file: %s", self.current_file)
except IOError as e:
log.error("Failed to create a new file! %s", e)
self.files.get_data_files().append(self.current_file)
self.current_file_records_count[0] = 0
try:
if self.buffered_writer is not None and self.buffered_writer.closed is False:
self.buffered_writer.close()
except IOError as e:
log.warning("Failed to close buffered writer! %s", e)
self.buffered_writer = None
try:
encoded = b64encode(msg.encode("utf-8"))
self.buffered_writer = self.get_or_init_buffered_writer(self.current_file)
self.buffered_writer.write(encoded + linesep.encode('utf-8'))
# self.buffered_writer.write(linesep.encode('utf-8'))
self.current_file_records_count[0] += 1
if self.current_file_records_count[0] - self.previous_file_records_count[0] >= self.settings.get_max_records_between_fsync():
self.previous_file_records_count = self.current_file_records_count[:]
self.buffered_writer.flush()
try:
if self.buffered_writer is not None and self.buffered_writer.closed is False:
self.buffered_writer.close()
except IOError as e:
log.warning("Failed to close buffered writer! %s", e)
except IOError as e:
log.warning("Failed to update data file![%s]\n%s", self.current_file, e)
def read_state_file(self):
try:
state_data_node = {}
try:
with BufferedReader(FileIO(self.settings.get_data_folder_path() +
self.files.get_state_file(), 'r')) as br:
state_data_node = load(br)
except JSONDecodeError:
log.error("Failed to decode JSON from state file")
state_data_node = 0
except IOError as e:
log.warning("Failed to fetch info from state file!", e)
reader_file = None
reader_pos = 0
if state_data_node:
reader_pos = state_data_node['position']
for file in sorted(self.files.get_data_files()):
if file == state_data_node['file']:
reader_file = file
break
if reader_file is None:
reader_file = sorted(self.files.get_data_files())[0]
reader_pos = 0
log.info("FileStorage_reader -- Initializing from state file: [%s:%i]",
self.settings.get_data_folder_path() + reader_file,
reader_pos)
return EventStorageReaderPointer(reader_file, reader_pos)
except Exception as e:
log.exception(e)
def get_or_init_buffered_reader(self, pointer):
try:
if self.buffered_reader is None or self.buffered_reader.closed:
new_file_to_read_path = self.settings.get_data_folder_path(
) + pointer.get_file()
# if not exists(new_file_to_read_path):
# next_file = self.get_next_file(self.files, self.new_pos)
# if next_file is not None:
# new_file_to_read_path = self.settings.get_data_folder_path() + next_file
# else:
# self.buffered_reader = None
# return None
self.buffered_reader = BufferedReader(
FileIO(new_file_to_read_path, 'r'))
lines_to_skip = pointer.get_line()
if lines_to_skip > 0:
while self.buffered_reader.readline() is not None:
if lines_to_skip > 0:
lines_to_skip -= 1
else:
break
return self.buffered_reader
except IOError as e:
log.error("Failed to initialize buffered reader! Error: %s", e)
raise RuntimeError("Failed to initialize buffered reader!", e)
except Exception as e:
log.exception(e)
def create_file(self, prefix, filename):
full_file_name = "%s%s.txt" % (prefix, filename)
file_path = "%s%s" % (self.settings.get_data_folder_path(), full_file_name)
try:
os_open(file_path, O_CREAT | O_EXCL)
return full_file_name
except IOError as e:
log.error("Failed to create a new file! Error: %s", e)
def get_or_init_buffered_writer(self, file):
try:
if self.buffered_writer is None or self.buffered_writer.closed:
self.buffered_writer = BufferedWriter(FileIO(self.settings.get_data_folder_path() + file, 'a'))
return self.buffered_writer
except IOError as e:
log.error("Failed to initialize buffered writer! Error: %s", e)
raise RuntimeError("Failed to initialize buffered writer!", e)
def write(self, msg):
if len(self.files.data_files) <= self.settings.get_max_files_count():
if self.current_file_records_count[
0] >= self.settings.get_max_records_per_file(
) or not exists(self.settings.get_data_folder_path() +
self.current_file):
try:
self.current_file = self.create_datafile()
log.debug(
"FileStorage_writer -- Created new data file: %s",
self.current_file)
except IOError as e:
log.error("Failed to create a new file! %s", e)
self.files.get_data_files().append(self.current_file)
self.current_file_records_count[0] = 0
try:
if self.buffered_writer is not None and self.buffered_writer.closed is False:
self.buffered_writer.close()
except IOError as e:
log.warning("Failed to close buffered writer! %s", e)
self.buffered_writer = None
try:
encoded = b64encode(msg.encode("utf-8"))
if not exists(self.settings.get_data_folder_path() +
self.current_file):
self.current_file = self.create_datafile()
self.buffered_writer = self.get_or_init_buffered_writer(
self.current_file)
self.buffered_writer.write(encoded + linesep.encode('utf-8'))
self.current_file_records_count[0] += 1
if self.current_file_records_count[
0] - self.previous_file_records_count[
0] >= self.settings.get_max_records_between_fsync(
):
self.previous_file_records_count = self.current_file_records_count[:]
self.buffered_writer.flush()
try:
if self.buffered_writer is not None and self.buffered_writer.closed is False:
self.buffered_writer.close()
except IOError as e:
log.warning("Failed to close buffered writer! %s", e)
except IOError as e:
log.warning("Failed to update data file![%s]\n%s",
self.current_file, e)
else:
raise DataFileCountError(
"The number of data files has been exceeded - change the settings or check the connection. New data will be lost."
)
def get_or_init_buffered_reader(self, pointer):
try:
if self.buffered_reader is None or self.buffered_reader.closed:
new_file_to_read_path = self.settings.get_data_folder_path() + pointer.get_file()
self.buffered_reader = BufferedReader(FileIO(new_file_to_read_path, 'r'))
lines_to_skip = pointer.get_line()
if lines_to_skip > 0:
while self.buffered_reader.readline() is not None:
if lines_to_skip != 0:
lines_to_skip -= 1
else:
break
return self.buffered_reader
except IOError as e:
log.error("Failed to initialize buffered reader!", e)
raise RuntimeError("Failed to initialize buffered reader!", e)
