def test_create_zip_with_attachments(self):
file = BytesIO()
file_content = b'file_content'
with patch('openassessment.data.OraDownloadData._download_file_by_key',
return_value=file_content) as download_mock:
OraDownloadData.create_zip_with_attachments(
file, COURSE_ID, self.submission_files_data)
download_mock.assert_has_calls([
call(self.file_key_5),
call(self.file_key_4),
call(self.file_key_1),
call(self.file_key_2),
call(self.file_key_3),
])
zip_file = zipfile.ZipFile(file)
# archive should contain five attachments, three parts text file and one csv
self.assertEqual(len(zip_file.infolist()), 9)
# check for pre_file_name_user's file and text
self.assertEqual(
zip_file.read(self.submission_files_data[0]['file_path']),
file_content)
self.assertEqual(
zip_file.read(self.submission_files_data[1]['file_path']),
self.answer_text.encode('utf-8'))
# check for pre_file_size_user's file and text
self.assertEqual(
zip_file.read(self.submission_files_data[2]['file_path']),
file_content)
self.assertEqual(
zip_file.read(self.submission_files_data[3]['file_path']),
self.answer_text.encode('utf-8'))
# check that main user's attachments have been written to the archive
self.assertEqual(
zip_file.read(self.submission_files_data[4]['file_path']),
file_content)
self.assertEqual(
zip_file.read(self.submission_files_data[5]['file_path']),
file_content)
self.assertEqual(
zip_file.read(self.submission_files_data[6]['file_path']),
file_content)
# main user's text response
self.assertEqual(
zip_file.read(self.submission_files_data[7]['file_path']),
self.answer_text.encode('utf-8'))
self.assertTrue(zip_file.read(os.path.join(COURSE_ID,
'downloads.csv')))
def test_collect_ora2_submission_files(self):
submission = sub_api._get_submission_model(self.submission['uuid']) # pylint: disable=protected-access
submission.answer = self.answer
submission.save()
# older submission formats
pre_filesize_uuid = self.pre_file_size_submission['uuid']
pre_file_size_submission = sub_api._get_submission_model(
pre_filesize_uuid) # pylint: disable=protected-access
pre_file_size_submission.answer = self.pre_file_size_answer
pre_file_size_submission.save()
pre_filename_uuid = self.pre_file_name_submission['uuid']
pre_file_name_submission = sub_api._get_submission_model(
pre_filename_uuid) # pylint: disable=protected-access
pre_file_name_submission.answer = self.pre_file_name_answer
pre_file_name_submission.save()
# answer for scorer submission is just a string, and `collect_ora2_submission_files`
# raises exception because of it, so we change it to empty dict
scorer_submission = sub_api._get_submission_model( # pylint: disable=protected-access
self.scorer_submission['uuid'])
scorer_submission.answer = {}
scorer_submission.save()
collected_ora_files_data = list(
OraDownloadData.collect_ora2_submission_files(COURSE_ID))
assert collected_ora_files_data == self.submission_files_data
def test_collect_ora2_submission_files(self):
submission = sub_api._get_submission_model(self.submission['uuid']) # pylint: disable=protected-access
submission.answer = self.answer
submission.save()
# answer for scorer submission is just a string, and `collect_ora2_submission_files`
# raises exception because of it, so we change it to empty dict
scorer_submission = sub_api._get_submission_model( # pylint: disable=protected-access
self.scorer_submission['uuid'])
scorer_submission.answer = {}
scorer_submission.save()
assert list(OraDownloadData.collect_ora2_submission_files(
COURSE_ID)) == self.submission_files_data
def upload_ora2_submission_files(_xmodule_instance_args, _entry_id, course_id,
_task_input, action_name):
"""
Creates zip archive with submission files in three steps:
1. Collect all files information using ORA download helper.
2. Download all submission attachments, put them in temporary zip
file along with submission texts and csv downloads list.
3. Upload zip file into reports storage.
"""
start_time = time()
start_date = datetime.now(UTC)
num_attempted = 1
num_total = 1
fmt = 'Task: {task_id}, InstructorTask ID: {entry_id}, Course: {course_id}, Input: {task_input}'
task_info_string = fmt.format(
task_id=_xmodule_instance_args.get('task_id')
if _xmodule_instance_args is not None else None,
entry_id=_entry_id,
course_id=course_id,
task_input=_task_input)
TASK_LOG.info('%s, Task type: %s, Starting task execution',
task_info_string, action_name)
task_progress = TaskProgress(action_name, num_total, start_time)
task_progress.attempted = num_attempted
step_manager = _task_step(task_progress, task_info_string, action_name)
submission_files_data = None
with step_manager(
'Collecting attachments data',
'Failed to get ORA submissions attachments data.',
'Error while collecting data',
):
submission_files_data = OraDownloadData.collect_ora2_submission_files(
course_id)
if submission_files_data is None:
return UPDATE_STATUS_FAILED
with TemporaryFile('rb+') as zip_file:
compressed = None
with step_manager(
'Downloading and compressing attachments files',
'Failed to download and compress submissions attachments.',
'Error while downloading and compressing submissions attachments',
):
compressed = OraDownloadData.create_zip_with_attachments(
zip_file, course_id, submission_files_data)
if compressed is None:
return UPDATE_STATUS_FAILED
zip_filename = None
with step_manager(
'Uploading zip file to storage',
'Failed to upload zip file to storage.',
'Error while uploading zip file to storage',
):
zip_filename = upload_zip_to_report_store(zip_file,
'submission_files',
course_id, start_date), # lint-amnesty, pylint: disable=trailing-comma-tuple
if not zip_filename:
return UPDATE_STATUS_FAILED
task_progress.succeeded = 1
curr_step = {'step': 'Finalizing attachments extracting'}
task_progress.update_task_state(extra_meta=curr_step)
TASK_LOG.info('%s, Task type: %s, Upload complete.', task_info_string,
action_name)
return UPDATE_STATUS_SUCCEEDED