Example #1
0
    def read_dataset_from_file_and_scan(self):
        """
        Reads file paths form a given file and returns a subset of them
        in a list.
        """
        # Set up logger and handler class.
        self.prepare_logging_rdf()
        self.logger.debug("***Scanning started.***")
        self.handler_factory_inst = handler_picker.HandlerPicker(self.configuration)
        self.handler_factory_inst.get_configured_handlers()

        file_containing_paths = self.conf("filename")
        start_file = self.conf("start")
        num_of_files = self.conf("num-files")

        self.logger.debug(
            "Copying paths from file {} start is {} and number of lines is {}.".format(
                file_containing_paths, start_file, num_of_files
            )
        )

        filename = os.path.basename(file_containing_paths)
        self.dataset_id = os.path.splitext(filename)[0]
        self.logger.debug("Dataset id is  {}.".format(self.dataset_id))

        content = util.read_file_into_list(file_containing_paths)

        self.total_number_of_files = len(content)
        self.logger.debug("{} lines read from file {}.".format((len(content)), file_containing_paths))

        if int(start_file) < 0 or int(start_file) > self.total_number_of_files:
            self.logger.error("Please correct start parameter value.")
            return

        end_file = int(start_file) + int(num_of_files)
        if end_file > self.total_number_of_files:
            self.logger.error("Please correct num-files parameter value because it is out of range.")
            return

        file_list = content[int(start_file) : end_file]
        content = None

        self.logger.debug("{} files copied in local file list.".format(len(file_list)))

        for path in file_list:
            self.file_list.append(path.rstrip())

        # at the end extract metadata.
        self.scan_files()
Example #2
0
def display_stats():

    cfg = read_cfg()

    filename =  os.path.join(cfg["core"]["log-path"], "fbs-stats.txt")

    file_contents = util.read_file_into_list(filename)

    x_axis = []
    y_axis = []
    for item in file_contents:
        values = item.split(",")
        x_axis.append(values[1])
        y_axis.append(int(values[3].rstrip()))

    plot(x_axis, y_axis)