def print_thread(self,
thread: int,
item: Any,
output_file: TextIO = sys.stderr) -> None:
with self.lock:
for_print = 'Thread %i: ' % (thread + 1)
diff = (self.__num_threads - thread)
print('\r'
'\033[%iA'
'%s'
'%s'
'\033[K'
'\r'
'\033[%iB' %
(diff, for_print, trim_to(
item, available_columns(for_print)), diff),
end='\r',
file=output_file)
def index_files(
input_dir: Path,
dat_file: Path) -> Dict[str, Optional[Path]]:
result: Dict[str, Optional[Path]] = {}
also_check_archive: bool = False
root = datafile.parse(dat_file, silence=True)
global RULES
if not RULES:
RULES = get_header_rules(root)
for game in root.game:
for rom_entry in game.rom:
result[rom_entry.sha1.lower()] = None
also_check_archive |= bool(ZIP_REGEX.search(rom_entry.name))
print('Scanning directory: %s\033[K' % input_dir, file=sys.stderr)
files_data = []
for full_path in input_dir.rglob('*'):
if not full_path.is_file():
continue
try:
print(
'%s%s\033[K' % (
FOUND_PREFIX,
trim_to(
full_path.relative_to(input_dir),
available_columns(FOUND_PREFIX) - 2)),
end='\r',
file=sys.stderr)
file_size = full_path.stat().st_size
files_data.append(FileData(file_size, full_path))
except OSError as e:
print(
'Error while reading file: %s\033[K' % e,
file=sys.stderr)
files_data.sort(key=FileData.get_size, reverse=True)
print('%s%i files\033[K' % (FOUND_PREFIX, len(files_data)), file=sys.stderr)
if files_data:
global PROGRESSBAR
PROGRESSBAR = MultiThreadedProgressBar(
len(files_data),
THREADS,
prefix='Calculating hashes')
PROGRESSBAR.init()
def process_thread_with_progress(
shared_files_data: List[FileData],
shared_result_data: List[Dict[str, Path]]) -> None:
curr_thread = current_thread()
if not isinstance(curr_thread, IndexedThread):
sys.exit('Bad thread type. Expected %s' % IndexedThread)
while True:
try:
next_file = shared_files_data.pop(0)
PROGRESSBAR.print_thread(
curr_thread.index,
next_file.path.relative_to(input_dir))
shared_result_data.append(process_file(
next_file,
also_check_archive))
PROGRESSBAR.print_bar()
except IndexError:
PROGRESSBAR.print_thread(curr_thread.index, "DONE")
break
threads = []
intermediate_results = []
for i in range(0, THREADS):
t = IndexedThread(
index=i,
target=process_thread_with_progress,
args=[files_data, intermediate_results],
daemon=True)
t.start()
threads.append(t)
for t in threads:
t.join()
print('\n', file=sys.stderr)
for intermediate_result in intermediate_results:
for key, value in intermediate_result.items():
if key in result and not \
(result[key] and is_zipfile(result[key])):
result[key] = value
return result