def test_results_aggregator(cleanup):
"""Test ResultsAggregator"""
if os.path.exists(OUTPUT):
shutil.rmtree(OUTPUT)
os.makedirs(os.path.join(OUTPUT, RESULTS_DIR))
results = [create_result(i) for i in range(100)]
batch1_file = get_results_temp_filename(OUTPUT, 1)
batch2_file = get_results_temp_filename(OUTPUT, 2)
pytest.aggregator1 = ResultsAggregator(batch1_file)
pytest.aggregator2 = ResultsAggregator(batch2_file)
pytest.aggregator1.create_file()
pytest.aggregator2.create_file()
assert os.path.exists(pytest.aggregator1._filename)
assert os.path.exists(pytest.aggregator2._filename)
with ProcessPoolExecutor() as executor:
executor.map(append, results)
final_results1 = pytest.aggregator1.get_results()
final_results1.sort(key=lambda x: int(x.name))
final_results2 = pytest.aggregator2.get_results()
final_results2.sort(key=lambda x: int(x.name))
expected1 = [x for x in results if int(x.name) % 2 == 0]
expected2 = [x for x in results if int(x.name) % 2 != 0]
assert final_results1 == expected1
assert final_results2 == expected2
results_dir = os.path.join(OUTPUT, RESULTS_DIR)
summary = ResultsAggregatorSummary(results_dir)
final_results = summary.get_results()
final_results.sort(key=lambda x: int(x.name))
assert final_results == results
summary.delete_files()
assert not [x for x in os.listdir(results_dir) if x.endswith(".csv")]
def _generate_jobs(self, config_file, verbose):
job_exec_class = self._config.job_execution_class()
results_filename = get_results_temp_filename(self._output,
self._batch_id)
results_aggregator = ResultsAggregator(results_filename)
results_aggregator.create_file()
return [
DispatchableJob(
job,
job_exec_class.generate_command(job,
self._jobs_output,
config_file,
verbose=verbose), self._output,
results_filename) for job in self._config.iter_jobs()
]