def test_reduce_csv_stream_resume(self, mock_open, mock_is_file, mock_read_csv, mock_to_csv, mock_pbar):
'''Test resuming works'''
mock_is_file.return_value = True
mock_read_csv.side_effect = [
self.extracted_dataframe_question,
self.reduced_dataframe_question_stream_partial,
self.reduced_dataframe_question_stream
]
output_file_name = reduce_panoptes_csv.reduce_csv(
self.extracted_csv_question,
self.config_yaml_question,
filter='all',
stream=True,
cpu_count=1
)
output_path = os.path.join(os.getcwd(), 'question_reducer_reductions.csv')
mock_is_file.assert_called_once_with(output_path)
self.assertEqual(output_file_name, output_path)
to_csv_calls = [
call(output_path, mode='a', index=False, header=False, encoding='utf-8'),
call(output_path, index=False, encoding='utf-8')
]
mock_to_csv.assert_has_calls(to_csv_calls, any_order=False)
mock_read_csv.assert_has_calls([call(output_path, encoding='utf-8')])
result_dataframe = reduce_panoptes_csv.flatten_data.return_values[0]
assert_frame_equal(result_dataframe, self.reduced_dataframe_question, check_like=True)
def test_reduce_csv_object_multi(self, mock_to_csv, mock_pbar):
'''Test object reducer makes one csv file with cpu_count=2'''
output_file_name = reduce_panoptes_csv.reduce_csv(
self.extracted_csv_question,
self.config_yaml_question,
filter='all',
cpu_count=2
)
output_path = os.path.join(os.getcwd(), 'question_reducer_reductions.csv')
self.assertEqual(output_file_name, output_path)
mock_to_csv.assert_called_once_with(output_path, index=False, encoding='utf-8')
def test_reduce_csv_list(self, mock_to_csv, mock_pbar):
'''Test list reducer makes one csv file'''
output_file_name = reduce_panoptes_csv.reduce_csv(
self.extracted_csv_survey,
self.config_yaml_survey,
order=True,
cpu_count=1
)
output_path = os.path.join(os.getcwd(), 'survey_reducer_reductions.csv')
self.assertEqual(output_file_name, output_path)
result_dataframe = reduce_panoptes_csv.order_columns.return_values[0]
assert_frame_equal(result_dataframe, self.reduced_dataframe_survey)
mock_to_csv.assert_called_once_with(output_path, index=False, encoding='utf-8')
def test_reduce_csv_object(self, mock_to_csv, mock_pbar):
'''Test object reducer makes one csv file'''
output_file_name = reduce_panoptes_csv.reduce_csv(
self.extracted_csv_question,
self.config_yaml_question,
filter='all',
cpu_count=1
)
mock_question_reducer.assert_any_call([{'yes': 1.0}, {'no': 1.0}], user_id=[1, 2])
output_path = os.path.join(os.getcwd(), 'question_reducer_reductions.csv')
self.assertEqual(output_file_name, output_path)
result_dataframe = reduce_panoptes_csv.flatten_data.return_values[0]
assert_frame_equal(result_dataframe, self.reduced_dataframe_question, check_like=True)
mock_to_csv.assert_called_once_with(output_path, index=False, encoding='utf-8')
def test_reduce_csv_stream_resume_nothing_new_to_do(self, mock_open, mock_is_file, mock_read_csv, mock_to_csv, mock_pbar):
'''Test resuming a finished file does not change it'''
mock_is_file.return_value = True
mock_read_csv.side_effect = [
self.extracted_dataframe_question,
self.reduced_dataframe_question,
self.reduced_dataframe_question
]
output_file_name = reduce_panoptes_csv.reduce_csv(
self.extracted_csv_question,
self.config_yaml_question,
filter='all',
stream=True,
cpu_count=1
)
output_path = os.path.join(os.getcwd(), 'question_reducer_reductions.csv')
mock_is_file.assert_called_once_with(output_path)
self.assertEqual(output_file_name, output_path)
mock_read_csv.assert_has_calls([call(output_path, encoding='utf-8')])
mock_to_csv.assert_not_called()