def test_process_extracted_file(self, process_exam_file_mock,
process_vcdc_file_mock,
process_eac_file_mock):
"""
Test that process_extracted_file handles file types correctly
"""
extracted_file = Mock()
process_eac_file_mock.return_value = (True, ['EAC'])
process_vcdc_file_mock.return_value = (True, ['VCDC'])
process_exam_file_mock.return_value = (True, ['EXAM'])
processor = download.ArchivedResponseProcessor(self.sftp)
assert processor.process_extracted_file(
extracted_file, 'eac-07-04-2016.dat') == (True, ['EAC'])
processor.process_eac_file.assert_called_once_with(extracted_file)
assert processor.process_extracted_file(
extracted_file, 'vcdc-07-04-2016.dat') == (True, ['VCDC'])
processor.process_vcdc_file.assert_called_once_with(extracted_file)
assert processor.process_extracted_file(
extracted_file, 'exam-07-04-2016.dat') == (True, ['EXAM'])
processor.process_exam_file.assert_called_once_with(extracted_file)
assert processor.process_extracted_file(
extracted_file, 'notatype-07-04-2016.dat') == (False, [])
def test_process_success(self, process_zip_mock, filtered_files_mock,
os_path_exists_mock, os_remove_mock):
"""Test the happy path"""
processor = download.ArchivedResponseProcessor(self.sftp)
processor.process()
filtered_files_mock.assert_called_once_with()
self.sftp.remove.assert_called_once_with('a.zip')
process_zip_mock.assert_called_once_with('/tmp/a.zip')
os_path_exists_mock.assert_called_once_with('/tmp/a.zip')
os_remove_mock.assert_called_once_with('/tmp/a.zip')
def test_get_invalid_row_messages(self):
"""Test generation of error messages"""
processor = download.ArchivedResponseProcessor(self.sftp)
messages = processor.get_invalid_row_messages([{
'Prop1': 'str',
'Prop2': 'bad_int',
}])
for msg in messages:
assert msg.startswith('Unable to parse row')
def test_process_failure(self, process_zip_mock, filtered_files_mock,
os_path_exists_mock, os_remove_mock):
"""Test the unhappy path"""
process_zip_mock.return_value = False
processor = download.ArchivedResponseProcessor(self.sftp)
processor.process()
filtered_files_mock.assert_called_once_with()
self.sftp.remove.assert_not_called()
process_zip_mock.assert_called_once_with('/tmp/a.zip')
os_path_exists_mock.assert_called_once_with('/tmp/a.zip')
os_remove_mock.assert_called_once_with('/tmp/a.zip')
def test_fetch_file(self):
"""
Tests that fetch_file works as expected
"""
remote_path = 'file.ext'
expected_local_path = '/tmp/file.ext'
processor = download.ArchivedResponseProcessor(self.sftp)
local_path = processor.fetch_file(remote_path)
assert local_path == expected_local_path
self.sftp.get.assert_called_once_with(remote_path,
localpath=expected_local_path)
def test_process_missing_local(self, process_zip_mock, filtered_files_mock,
os_path_exists_mock, os_remove_mock):
"""Test that a missing local file doesn't fail"""
os_path_exists_mock.return_value = False
processor = download.ArchivedResponseProcessor(self.sftp)
processor.process()
filtered_files_mock.assert_called_once_with()
self.sftp.remove.assert_called_once_with('a.zip')
process_zip_mock.assert_called_once_with('/tmp/a.zip')
os_path_exists_mock.assert_called_once_with('/tmp/a.zip')
os_remove_mock.assert_not_called()
def test_process_exception(self, process_zip_mock, filtered_files_mock,
os_path_exists_mock, os_remove_mock):
"""Test that process() cleans up the local but not the remote on any processing exception"""
process_zip_mock.side_effect = Exception('exception')
processor = download.ArchivedResponseProcessor(self.sftp)
processor.process()
filtered_files_mock.assert_called_once_with()
self.sftp.remove.assert_not_called()
process_zip_mock.assert_called_once_with('/tmp/a.zip')
os_path_exists_mock.assert_called_once_with('/tmp/a.zip')
os_remove_mock.assert_called_once_with('/tmp/a.zip')
def test_process_ssh_exception_cd(self, process_zip_mock,
filtered_files_mock, os_path_exists_mock,
os_remove_mock):
"""Test that SSH exceptions bubble up"""
self.sftp.cd.side_effect = SSHException('exception')
processor = download.ArchivedResponseProcessor(self.sftp)
with self.assertRaises(RetryableSFTPException):
processor.process()
filtered_files_mock.assert_not_called()
self.sftp.remove.assert_not_called()
process_zip_mock.assert_not_called()
os_path_exists_mock.assert_not_called()
os_remove_mock.assert_not_called()
def test_process_ssh_exception_remove(self, exc, process_zip_mock,
filtered_files_mock,
os_path_exists_mock, os_remove_mock):
"""Test that SSH exceptions bubble up"""
self.sftp.remove.side_effect = exc
processor = download.ArchivedResponseProcessor(self.sftp)
with self.assertRaises(RetryableSFTPException):
processor.process()
filtered_files_mock.assert_called_once_with()
self.sftp.remove.assert_called_once_with('a.zip')
process_zip_mock.assert_called_once_with('/tmp/a.zip')
os_path_exists_mock.assert_called_once_with('/tmp/a.zip')
os_remove_mock.assert_called_once_with('/tmp/a.zip')
def test_process_zip_email(self, process_extracted_file_mock,
zip_file_mock):
"""Tests that an email is sent if errors returned"""
process_extracted_file_mock.return_value = (True, ['ERROR'])
zip_file_mock.return_value.__enter__.return_value.namelist.return_value = [
'a.dat'
]
with patch('exams.pearson.utils.email_processing_failures'
) as email_processing_failures_mock:
processor = download.ArchivedResponseProcessor(self.sftp)
processor.process_zip('local.zip')
self.auditor.return_value.audit_response_file.assert_called_once_with(
'local.zip')
email_processing_failures_mock.assert_called_once_with(
'a.dat', 'local.zip', ['ERROR'])
def test_process_zip(self, files, results, expected_result,
process_extracted_file_mock, zip_file_mock):
"""Tests that process_zip behaves correctly"""
process_extracted_file_mock.side_effect = results
zip_file_mock.return_value.__enter__.return_value.namelist.return_value = files
with patch('exams.pearson.utils.email_processing_failures'
) as email_processing_failures_mock, patch(
'exams.pearson.download.locally_extracted'
) as locally_extracted_mock:
locally_extracted_mock.__enter__.return_value = []
processor = download.ArchivedResponseProcessor(self.sftp)
assert processor.process_zip('local.zip') == expected_result
self.auditor.return_value.audit_response_file.assert_called_once_with(
'local.zip')
email_processing_failures_mock.assert_not_called()
def test_filtered_files(self):
"""
Test that filtered_files filters on the regex
"""
listdir_values = ['a.zip', 'b.zip', 'b']
isfile_values = [True, False, True]
self.sftp.listdir.return_value = listdir_values
self.sftp.isfile.side_effect = isfile_values
processor = download.ArchivedResponseProcessor(self.sftp)
result = list(processor.filtered_files())
assert result == [('a.zip', '/tmp/a.zip')]
self.sftp.listdir.assert_called_once_with()
assert self.sftp.isfile.call_args_list == [
call(arg) for arg in listdir_values
]
def batch_process_pearson_zip_files():
"""
Fetch zip files from pearsons sftp periodically.
"""
if not settings.FEATURES.get('PEARSON_EXAMS_SYNC', False):
log.info(
'Feature PEARSON_EXAM_SYNC disabled, not executing batch_process_pearson_zip_files'
)
return
try:
with sftp.get_connection() as sftp_connection:
processor = download.ArchivedResponseProcessor(sftp_connection)
processor.process()
except ImproperlyConfigured:
log.exception(
'PEARSON_EXAMS_SYNC enabled, but not configured correctly')
except RetryableSFTPException:
log.exception('Retryable error during SFTP operation')
def setUpTestData(cls):
sftp = Mock()
cls.processor = download.ArchivedResponseProcessor(sftp)
cls.course = course = CourseFactory.create()
cls.success_auths = ExamAuthorizationFactory.create_batch(
2, course=course)
cls.failure_auths = ExamAuthorizationFactory.create_batch(
2, course=course)
cls.success_results = ([
EACResult(
client_authorization_id=auth.id,
client_candidate_id=auth.user.profile.student_id,
date=FIXED_DATETIME,
status=EAC_SUCCESS_STATUS,
message='',
) for auth in cls.success_auths
], [])
cls.failed_results = ([
EACResult(
client_authorization_id=cls.failure_auths[0].id,
client_candidate_id=cls.failure_auths[0].user.profile.
student_id,
date=FIXED_DATETIME,
status=EAC_FAILURE_STATUS,
message='',
),
EACResult(
client_authorization_id=cls.failure_auths[1].id,
client_candidate_id=cls.failure_auths[1].user.profile.
student_id,
date=FIXED_DATETIME,
status=EAC_FAILURE_STATUS,
message='wrong username',
),
], [])
cls.all_results = (
cls.success_results[0] + cls.failed_results[0],
cls.success_results[1] + cls.failed_results[1],
)
def setUpTestData(cls):
sftp = Mock()
cls.now = now_in_utc()
cls.processor = download.ArchivedResponseProcessor(sftp)
with mute_signals(post_save):
cls.success_profiles = ExamProfileFactory.create_batch(2) + [
ExamProfileFactory.create(
profile__id=999,
profile__student_id=1000), # disjoint id and student_id
]
cls.failure_profiles = ExamProfileFactory.create_batch(2)
cls.success_results = ([
VCDCResult(
client_candidate_id=exam_profile.profile.student_id,
status=VCDC_SUCCESS_STATUS,
date=cls.now,
message='',
) for exam_profile in cls.success_profiles
], [])
cls.failed_results = ([
VCDCResult(
client_candidate_id=cls.failure_profiles[0].profile.student_id,
status=VCDC_FAILURE_STATUS,
date=cls.now,
message='',
),
VCDCResult(
client_candidate_id=cls.failure_profiles[1].profile.student_id,
status=VCDC_FAILURE_STATUS,
date=cls.now,
message='Bad address',
),
], [])
cls.all_results = (
cls.success_results[0] + cls.failed_results[0],
cls.success_results[1] + cls.failed_results[1],
)
def setUpTestData(cls):
cls.processor = download.ArchivedResponseProcessor(Mock())
cls.course = CourseFactory.create()