def test_util(fname): lines = Utility.file_len(fname) assert lines == 64 lines, split_list = Utility.split_files(fname, lines, 4) assert len(split_list) == 4 assert lines == 16 return lines, split_list
def data_process(self): """ This method is to process the data. It will calculate the line count in file and split file based on line count. Now files are split to the number based on number of cores in machine. returns :Nothing """ logging.info('Processing the data and split files') lines = Utility.file_len(self.fname) self.lines_to_be, self.split_files = Utility.split_files(self.fname, lines, cpu_count().real)