def test_process_extracted_file(self, process_exam_file_mock, process_vcdc_file_mock, process_eac_file_mock): """ Test that process_extracted_file handles file types correctly """ extracted_file = Mock() process_eac_file_mock.return_value = (True, ['EAC']) process_vcdc_file_mock.return_value = (True, ['VCDC']) process_exam_file_mock.return_value = (True, ['EXAM']) processor = download.ArchivedResponseProcessor(self.sftp) assert processor.process_extracted_file( extracted_file, 'eac-07-04-2016.dat') == (True, ['EAC']) processor.process_eac_file.assert_called_once_with(extracted_file) assert processor.process_extracted_file( extracted_file, 'vcdc-07-04-2016.dat') == (True, ['VCDC']) processor.process_vcdc_file.assert_called_once_with(extracted_file) assert processor.process_extracted_file( extracted_file, 'exam-07-04-2016.dat') == (True, ['EXAM']) processor.process_exam_file.assert_called_once_with(extracted_file) assert processor.process_extracted_file( extracted_file, 'notatype-07-04-2016.dat') == (False, [])
def test_process_success(self, process_zip_mock, filtered_files_mock, os_path_exists_mock, os_remove_mock): """Test the happy path""" processor = download.ArchivedResponseProcessor(self.sftp) processor.process() filtered_files_mock.assert_called_once_with() self.sftp.remove.assert_called_once_with('a.zip') process_zip_mock.assert_called_once_with('/tmp/a.zip') os_path_exists_mock.assert_called_once_with('/tmp/a.zip') os_remove_mock.assert_called_once_with('/tmp/a.zip')
def test_get_invalid_row_messages(self): """Test generation of error messages""" processor = download.ArchivedResponseProcessor(self.sftp) messages = processor.get_invalid_row_messages([{ 'Prop1': 'str', 'Prop2': 'bad_int', }]) for msg in messages: assert msg.startswith('Unable to parse row')
def test_process_failure(self, process_zip_mock, filtered_files_mock, os_path_exists_mock, os_remove_mock): """Test the unhappy path""" process_zip_mock.return_value = False processor = download.ArchivedResponseProcessor(self.sftp) processor.process() filtered_files_mock.assert_called_once_with() self.sftp.remove.assert_not_called() process_zip_mock.assert_called_once_with('/tmp/a.zip') os_path_exists_mock.assert_called_once_with('/tmp/a.zip') os_remove_mock.assert_called_once_with('/tmp/a.zip')
def test_fetch_file(self): """ Tests that fetch_file works as expected """ remote_path = 'file.ext' expected_local_path = '/tmp/file.ext' processor = download.ArchivedResponseProcessor(self.sftp) local_path = processor.fetch_file(remote_path) assert local_path == expected_local_path self.sftp.get.assert_called_once_with(remote_path, localpath=expected_local_path)
def test_process_missing_local(self, process_zip_mock, filtered_files_mock, os_path_exists_mock, os_remove_mock): """Test that a missing local file doesn't fail""" os_path_exists_mock.return_value = False processor = download.ArchivedResponseProcessor(self.sftp) processor.process() filtered_files_mock.assert_called_once_with() self.sftp.remove.assert_called_once_with('a.zip') process_zip_mock.assert_called_once_with('/tmp/a.zip') os_path_exists_mock.assert_called_once_with('/tmp/a.zip') os_remove_mock.assert_not_called()
def test_process_exception(self, process_zip_mock, filtered_files_mock, os_path_exists_mock, os_remove_mock): """Test that process() cleans up the local but not the remote on any processing exception""" process_zip_mock.side_effect = Exception('exception') processor = download.ArchivedResponseProcessor(self.sftp) processor.process() filtered_files_mock.assert_called_once_with() self.sftp.remove.assert_not_called() process_zip_mock.assert_called_once_with('/tmp/a.zip') os_path_exists_mock.assert_called_once_with('/tmp/a.zip') os_remove_mock.assert_called_once_with('/tmp/a.zip')
def test_process_ssh_exception_cd(self, process_zip_mock, filtered_files_mock, os_path_exists_mock, os_remove_mock): """Test that SSH exceptions bubble up""" self.sftp.cd.side_effect = SSHException('exception') processor = download.ArchivedResponseProcessor(self.sftp) with self.assertRaises(RetryableSFTPException): processor.process() filtered_files_mock.assert_not_called() self.sftp.remove.assert_not_called() process_zip_mock.assert_not_called() os_path_exists_mock.assert_not_called() os_remove_mock.assert_not_called()
def test_process_ssh_exception_remove(self, exc, process_zip_mock, filtered_files_mock, os_path_exists_mock, os_remove_mock): """Test that SSH exceptions bubble up""" self.sftp.remove.side_effect = exc processor = download.ArchivedResponseProcessor(self.sftp) with self.assertRaises(RetryableSFTPException): processor.process() filtered_files_mock.assert_called_once_with() self.sftp.remove.assert_called_once_with('a.zip') process_zip_mock.assert_called_once_with('/tmp/a.zip') os_path_exists_mock.assert_called_once_with('/tmp/a.zip') os_remove_mock.assert_called_once_with('/tmp/a.zip')
def test_process_zip_email(self, process_extracted_file_mock, zip_file_mock): """Tests that an email is sent if errors returned""" process_extracted_file_mock.return_value = (True, ['ERROR']) zip_file_mock.return_value.__enter__.return_value.namelist.return_value = [ 'a.dat' ] with patch('exams.pearson.utils.email_processing_failures' ) as email_processing_failures_mock: processor = download.ArchivedResponseProcessor(self.sftp) processor.process_zip('local.zip') self.auditor.return_value.audit_response_file.assert_called_once_with( 'local.zip') email_processing_failures_mock.assert_called_once_with( 'a.dat', 'local.zip', ['ERROR'])
def test_process_zip(self, files, results, expected_result, process_extracted_file_mock, zip_file_mock): """Tests that process_zip behaves correctly""" process_extracted_file_mock.side_effect = results zip_file_mock.return_value.__enter__.return_value.namelist.return_value = files with patch('exams.pearson.utils.email_processing_failures' ) as email_processing_failures_mock, patch( 'exams.pearson.download.locally_extracted' ) as locally_extracted_mock: locally_extracted_mock.__enter__.return_value = [] processor = download.ArchivedResponseProcessor(self.sftp) assert processor.process_zip('local.zip') == expected_result self.auditor.return_value.audit_response_file.assert_called_once_with( 'local.zip') email_processing_failures_mock.assert_not_called()
def test_filtered_files(self): """ Test that filtered_files filters on the regex """ listdir_values = ['a.zip', 'b.zip', 'b'] isfile_values = [True, False, True] self.sftp.listdir.return_value = listdir_values self.sftp.isfile.side_effect = isfile_values processor = download.ArchivedResponseProcessor(self.sftp) result = list(processor.filtered_files()) assert result == [('a.zip', '/tmp/a.zip')] self.sftp.listdir.assert_called_once_with() assert self.sftp.isfile.call_args_list == [ call(arg) for arg in listdir_values ]
def batch_process_pearson_zip_files(): """ Fetch zip files from pearsons sftp periodically. """ if not settings.FEATURES.get('PEARSON_EXAMS_SYNC', False): log.info( 'Feature PEARSON_EXAM_SYNC disabled, not executing batch_process_pearson_zip_files' ) return try: with sftp.get_connection() as sftp_connection: processor = download.ArchivedResponseProcessor(sftp_connection) processor.process() except ImproperlyConfigured: log.exception( 'PEARSON_EXAMS_SYNC enabled, but not configured correctly') except RetryableSFTPException: log.exception('Retryable error during SFTP operation')
def setUpTestData(cls): sftp = Mock() cls.processor = download.ArchivedResponseProcessor(sftp) cls.course = course = CourseFactory.create() cls.success_auths = ExamAuthorizationFactory.create_batch( 2, course=course) cls.failure_auths = ExamAuthorizationFactory.create_batch( 2, course=course) cls.success_results = ([ EACResult( client_authorization_id=auth.id, client_candidate_id=auth.user.profile.student_id, date=FIXED_DATETIME, status=EAC_SUCCESS_STATUS, message='', ) for auth in cls.success_auths ], []) cls.failed_results = ([ EACResult( client_authorization_id=cls.failure_auths[0].id, client_candidate_id=cls.failure_auths[0].user.profile. student_id, date=FIXED_DATETIME, status=EAC_FAILURE_STATUS, message='', ), EACResult( client_authorization_id=cls.failure_auths[1].id, client_candidate_id=cls.failure_auths[1].user.profile. student_id, date=FIXED_DATETIME, status=EAC_FAILURE_STATUS, message='wrong username', ), ], []) cls.all_results = ( cls.success_results[0] + cls.failed_results[0], cls.success_results[1] + cls.failed_results[1], )
def setUpTestData(cls): sftp = Mock() cls.now = now_in_utc() cls.processor = download.ArchivedResponseProcessor(sftp) with mute_signals(post_save): cls.success_profiles = ExamProfileFactory.create_batch(2) + [ ExamProfileFactory.create( profile__id=999, profile__student_id=1000), # disjoint id and student_id ] cls.failure_profiles = ExamProfileFactory.create_batch(2) cls.success_results = ([ VCDCResult( client_candidate_id=exam_profile.profile.student_id, status=VCDC_SUCCESS_STATUS, date=cls.now, message='', ) for exam_profile in cls.success_profiles ], []) cls.failed_results = ([ VCDCResult( client_candidate_id=cls.failure_profiles[0].profile.student_id, status=VCDC_FAILURE_STATUS, date=cls.now, message='', ), VCDCResult( client_candidate_id=cls.failure_profiles[1].profile.student_id, status=VCDC_FAILURE_STATUS, date=cls.now, message='Bad address', ), ], []) cls.all_results = ( cls.success_results[0] + cls.failed_results[0], cls.success_results[1] + cls.failed_results[1], )
def setUpTestData(cls): cls.processor = download.ArchivedResponseProcessor(Mock()) cls.course = CourseFactory.create()