def save_word_error_parallel(results_file, output_file, X_test, y_test):
file = open(results_file, 'r')
lines = file.readlines()
file.close()
pool = Pool(4) # uses up 4 cores per call (for 8 core machine), so change this to match half your cpu count
f_vals = []
# packs a tuple - line, index, X_test, y_test, verbose (should print or not)
data = [(l, i, X_test, y_test, True) for i, l in enumerate(lines)]
# process it in correct order
accuracies = pool.imap(process_line, data) # maintains order when processing in parallel
# wait while processing the data
pool.close()
for acc in accuracies:
f_vals.append(str(acc) + "\n")
file = open(output_file, 'w')
file.writelines(f_vals)
file.close()
def run(self):
pool = Pool(40)
return dict(pool.imap(self._run, self.runners))