def read_dataset_from_file_and_scan(self):
"""
Reads file paths form a given file and returns a subset of them
in a list.
"""
# Set up logger and handler class.
self.prepare_logging_rdf()
self.logger.debug("***Scanning started.***")
self.handler_factory_inst = handler_picker.HandlerPicker(self.configuration)
self.handler_factory_inst.get_configured_handlers()
file_containing_paths = self.conf("filename")
start_file = self.conf("start")
num_of_files = self.conf("num-files")
self.logger.debug(
"Copying paths from file {} start is {} and number of lines is {}.".format(
file_containing_paths, start_file, num_of_files
)
)
filename = os.path.basename(file_containing_paths)
self.dataset_id = os.path.splitext(filename)[0]
self.logger.debug("Dataset id is {}.".format(self.dataset_id))
content = util.read_file_into_list(file_containing_paths)
self.total_number_of_files = len(content)
self.logger.debug("{} lines read from file {}.".format((len(content)), file_containing_paths))
if int(start_file) < 0 or int(start_file) > self.total_number_of_files:
self.logger.error("Please correct start parameter value.")
return
end_file = int(start_file) + int(num_of_files)
if end_file > self.total_number_of_files:
self.logger.error("Please correct num-files parameter value because it is out of range.")
return
file_list = content[int(start_file) : end_file]
content = None
self.logger.debug("{} files copied in local file list.".format(len(file_list)))
for path in file_list:
self.file_list.append(path.rstrip())
# at the end extract metadata.
self.scan_files()
def display_stats():
cfg = read_cfg()
filename = os.path.join(cfg["core"]["log-path"], "fbs-stats.txt")
file_contents = util.read_file_into_list(filename)
x_axis = []
y_axis = []
for item in file_contents:
values = item.split(",")
x_axis.append(values[1])
y_axis.append(int(values[3].rstrip()))
plot(x_axis, y_axis)