def save_entries(file_path, entry_generator):
"""
Store the entries as a file. IDS entries in IDSE files, log entries as log files.
returns: The file path in which the file was saved.
"""
entries_list = list(entry_generator)
first_entry = entries_list[0]
# Where to store?
file_path_full = None
# What to store?
lines = None
# How to convert?
to_line = None
# LogEntry objects: No extension, no header, call entry.get_log_string()
if isinstance(first_entry, LogEntry):
file_path_full = file_path
lines = []
to_line = lambda l: l.get_log_string()
# IdsEntry objects: IDSE extension, IDSE header and run _ids_entry_to_idse_string(entry)
elif isinstance(first_entry, IdsEntry):
file_path_full = add_idse_extension(file_path)
lines = [HEADER]
to_line = _ids_entry_to_idse_string
else:
raise TypeError(
"[IDSE DAO] Given elements are neither LogEntry nor IdsEntry objects!"
)
if os.path.lexists(file_path_full):
_raise_file_exists(file_path_full)
# Actual entry -> string conversion
lines.extend([to_line(e) for e in entries_list])
Dir.write_lines(file_path_full, lines)
return file_path_full
def _sample(file_path, number_of_elements, limit_to):
""" Sample <number_of_elements> from the given file. """
print("Sampling...")
target_file_path = "%s_%s-sample" % (file_path, number_of_elements)
if not os.path.lexists(file_path):
raise IOError("Input file doesn't exist")
target_file_path = Dir.uniquify(target_file_path)
line_generator = Dir.yield_lines(file_path)
log_lines = None
if limit_to is None:
log_lines = ids_tools.reservoir_sample(line_generator, number_of_elements)
else:
log_lines = ids_tools.reservoir_sample_limit(line_generator, number_of_elements, limit_to)
Dir.write_lines(target_file_path, log_lines)
print("Done. Wrote to file:\n%s" % target_file_path)
def store_experiment(self):
""" Store the results saved in this class in our experiment directory. """
self.end_time = time.time()
self.storer_printer.prt("Storing experiment results...")
Dir.ensure_folder_exists(self.experiment_dir_path)
entry_file_path = os.path.join(self.experiment_dir_path,
"used_entries")
result_file_path = os.path.join(self.experiment_dir_path, "result")
stdout_file_path = os.path.join(self.experiment_dir_path, "stdout")
classifiers_file_path = os.path.join(self.experiment_dir_path,
"classifiers")
file_paths = [
entry_file_path, result_file_path, stdout_file_path,
classifiers_file_path
]
other_result_files_paths = []
for file_name, _ in self.other_result_files:
oth_res_path_creation = os.path.join(self.experiment_dir_path,
file_name)
oth_res_path_creation = Dir.uniquify(oth_res_path_creation)
other_result_files_paths.append(oth_res_path_creation)
if any([
os.path.lexists(x)
for x in file_paths + other_result_files_paths
]):
raise IOError("One of the files exists: %s" %
(file_paths + other_result_files_paths))
self.storer_printer.prt("Data verified. Storing utilised entries...")
# Create new file with my entries
saved_path = idse_dao.save_entries(entry_file_path, self.entries)
self.storer_printer.prt("Done. Analysing file...")
# Analyse that file
log_file_analysis.analyse(saved_path,
to_file=True,
output_printer=util.prtr.Storer())
self.storer_printer.prt("Done. Saving classifiers...")
# Save trained classifiers
classifier_lines = self.create_classifier_lines()
Dir.write_lines(classifiers_file_path, classifier_lines)
self.storer_printer.prt("Done. Saving result digest...")
# Save the result
result_lines = self.create_result_lines()
Dir.write_lines(result_file_path, result_lines)
if self.other_result_files:
for oth_res_path, (oth_res_name,
oth_res_lines) in zip(other_result_files_paths,
self.other_result_files):
self.storer_printer.prt("Saving others: %s..." % oth_res_name)
Dir.write_lines(oth_res_path, oth_res_lines)
self.storer_printer.prt("Done!")
self.storer_printer.prt("Experiment stored in: %s" %
self.experiment_dir_path)
# Save the stdout (tee replacement)
stdout_lines = self.storer_printer.get_messages()
Dir.write_lines(stdout_file_path, stdout_lines)