def word_count_dir(dir_path): file_names = [] db = DBConnect() dictionary = db.select_tags() for root, dirs, files in walk(dir_path): for name in files: if 'pdf' in name or 'htm' in name or 'txt' in name: file_names.append(path.join(root, name)) else: continue click.secho("Counting words for {} PDF documents.".format(len(file_names)) , fg='blue') n_jobs = 20 dir_counters = Parallel(n_jobs=n_jobs)(delayed(word_count)(file_name) for file_name in file_names) total = sum(dir_counters, Counter()) for word in dictionary: if word in total: db.insert_word({'word': word, 'frequency': total[word]}) return total