def push_student_responses_to_s3(_xmodule_instance_args, _entry_id, course_id, _task_input, action_name): """ For a given `course_id`, generate a responses CSV file for students that have submitted problem responses, and store using a `ReportStore`. Once created, the files can be accessed by instantiating another `ReportStore` (via `ReportStore.from_config()`) and calling `link_for()` on it. Writes are buffered, so we'll never write part of a CSV file to S3 -- i.e. any files that are visible in ReportStore will be complete ones. """ start_time = datetime.now(UTC) try: course = get_course_by_id(course_id) except ValueError as e: TASK_LOG.error(e.message) return "failed" rows = student_response_rows(course) # Generate parts of the file name timestamp_str = start_time.strftime("%Y-%m-%d-%H%M") course_id_prefix = course_filename_prefix_generator(course_id) # Perform the actual upload report_store = ReportStore.from_config(config_name='GRADES_DOWNLOAD') report_store.store_rows( course_id, u"{course_id_prefix}_responses_report_{timestamp_str}.csv".format( course_id_prefix=course_id_prefix, timestamp_str=timestamp_str, ), rows) return "succeeded"
def push_student_responses_to_s3(_xmodule_instance_args, _entry_id, course_id, _task_input, action_name): """ For a given `course_id`, generate a responses CSV file for students that have submitted problem responses, and store using a `ReportStore`. Once created, the files can be accessed by instantiating another `ReportStore` (via `ReportStore.from_config()`) and calling `link_for()` on it. Writes are buffered, so we'll never write part of a CSV file to S3 -- i.e. any files that are visible in ReportStore will be complete ones. """ start_time = datetime.now(UTC) try: course = get_course_by_id(course_id) except ValueError as e: TASK_LOG.error(e.message) return "failed" rows = student_response_rows(course) # Generate parts of the file name timestamp_str = start_time.strftime("%Y-%m-%d-%H%M") course_id_prefix = course_filename_prefix_generator(course_id) # Perform the actual upload report_store = ReportStore.from_config(config_name='GRADES_DOWNLOAD') report_store.store_rows( course_id, u"{course_id_prefix}_responses_report_{timestamp_str}.csv".format( course_id_prefix=course_id_prefix, timestamp_str=timestamp_str, ), rows ) return "succeeded"
def _upload_exec_summary_to_store(data_dict, report_name, course_id, generated_at, config_name='FINANCIAL_REPORTS'): """ Upload Executive Summary Html file using ReportStore. Arguments: data_dict: containing executive report data. report_name: Name of the resulting Html File. course_id: ID of the course """ report_store = ReportStore.from_config(config_name) # Use the data dict and html template to generate the output buffer output_buffer = StringIO( render_to_string( "instructor/instructor_dashboard_2/executive_summary.html", data_dict)) report_store.store( course_id, u"{course_prefix}_{report_name}_{timestamp_str}.html".format( course_prefix=course_filename_prefix_generator(course_id), report_name=report_name, timestamp_str=generated_at.strftime("%Y-%m-%d-%H%M")), output_buffer, ) tracker_emit(report_name)
def test_gradeucursos_post_from_instructor_tab_is_resumen(self): """ Test gradeucursos post from instructor tab normal process with is_resumen params """ self.skipTest("disabled temporarily") with mock_get_score(1, 2): self.grade_factory.update(self.student, self.course, force_update_subsections=True) self.grade_factory.update(self.student_2, self.course, force_update_subsections=True) try: from uchileedxlogin.models import EdxLoginUser EdxLoginUser.objects.create(user=self.student, run='09472337K') except ImportError: self.skipTest("import error uchileedxlogin") task_input = { 'grade_type': 'seven_scale', 'course_id': str(self.course.id), 'instructor_tab': True, 'assig_type': 'Homework', 'is_resumen': True } with patch('lms.djangoapps.instructor_task.tasks_helper.runner._get_current_task'): result = task_get_data( None, None, self.course.id, task_input, 'EOL_GRADE_UCURSOS' ) report_store = ReportStore.from_config(config_name='GRADES_DOWNLOAD') report_csv_filename = report_store.links_for(self.course.id)[0][0] report_path = report_store.path_to(self.course.id, report_csv_filename) self.assertTrue('_notas_estudiantes_' in report_csv_filename) self.assertTrue('_notas_estudiantes_' in report_path)
def verify_rows_in_csv(self, expected_rows, file_index=0, verify_order=True, ignore_other_columns=False): """ Verify that the last ReportStore CSV contains the expected content. Arguments: expected_rows (iterable): An iterable of dictionaries, where each dict represents a row of data in the last ReportStore CSV. Each dict maps keys from the CSV header to values in that row's corresponding cell. file_index (int): Describes which report store file to open. Files are ordered by last modified date, and 0 corresponds to the most recently modified file. verify_order (boolean): When True (default), we verify that both the content and order of `expected_rows` matches the actual csv rows. When False, we only verify that the content matches. ignore_other_columns (boolean): When True, we verify that `expected_rows` contain data which is the subset of actual csv rows. """ report_store = ReportStore.from_config(config_name='GRADES_DOWNLOAD') report_csv_filename = report_store.links_for(self.course.id)[file_index][0] report_path = report_store.path_to(self.course.id, report_csv_filename) with report_store.storage.open(report_path) as csv_file: # Expand the dict reader generator so we don't lose it's content csv_rows = [row for row in unicodecsv.DictReader(csv_file)] if ignore_other_columns: csv_rows = [ {key: row.get(key) for key in expected_rows[index].keys()} for index, row in enumerate(csv_rows) ] if verify_order: self.assertEqual(csv_rows, expected_rows) else: self.assertItemsEqual(csv_rows, expected_rows)
def test_xblockcompletion_get_all_data_no_responses(self, report): """ Test xblockcompletion view all data when xblock dont have responses yet """ generated_report_data = {} report.return_value = generated_report_data data = {'format': False, 'course': str(self.course.id), 'base_url':'this_is_a_url'} task_input = {'data': data } with patch('lms.djangoapps.instructor_task.tasks_helper.runner._get_current_task'): result = generate( None, None, self.course.id, task_input, 'EOL_Xblock_Completion' ) report_store = ReportStore.from_config(config_name='GRADES_DOWNLOAD') header_row = ",".join(['Titulo', 'Username', 'Email', 'Run', 'Seccion', 'SubSeccion', 'Unidad', 'Pregunta', 'Respuesta Estudiante', 'Resp. Correcta', 'Intentos', 'Pts Ganados', 'Pts Posibles', 'Pts Total Componente', 'Url', 'block_id']) base_student_row = ",".join([ self.items[0].display_name, self.student.username, self.student.email, '', '1.' + self.chapter.display_name, '1.1.' + self.section.display_name, '1.1.1.' + self.subsection.display_name ]) report_csv_filename = report_store.links_for(self.course.id)[0][0] report_path = report_store.path_to(self.course.id, report_csv_filename) with report_store.storage.open(report_path) as csv_file: csv_file_data = csv_file.read() # Removing unicode signature (BOM) from the beginning csv_file_data = csv_file_data.decode("utf-8-sig") self.assertIn(header_row, csv_file_data) self.assertFalse(base_student_row in csv_file_data)
def verify_rows_in_csv(self, expected_rows, file_index=0, verify_order=True, ignore_other_columns=False): """ Verify that the last ReportStore CSV contains the expected content. Arguments: expected_rows (iterable): An iterable of dictionaries, where each dict represents a row of data in the last ReportStore CSV. Each dict maps keys from the CSV header to values in that row's corresponding cell. file_index (int): Describes which report store file to open. Files are ordered by last modified date, and 0 corresponds to the most recently modified file. verify_order (boolean): When True (default), we verify that both the content and order of `expected_rows` matches the actual csv rows. When False, we only verify that the content matches. ignore_other_columns (boolean): When True, we verify that `expected_rows` contain data which is the subset of actual csv rows. """ report_store = ReportStore.from_config(config_name='GRADES_DOWNLOAD') report_csv_filename = report_store.links_for(self.course.id)[file_index][0] report_path = report_store.path_to(self.course.id, report_csv_filename) with report_store.storage.open(report_path) as csv_file: # Expand the dict reader generator so we don't lose it's content csv_rows = [row for row in unicodecsv.DictReader(csv_file, encoding='utf-8-sig')] if ignore_other_columns: csv_rows = [ {key: row.get(key) for key in expected_rows[index].keys()} for index, row in enumerate(csv_rows) ] if verify_order: self.assertEqual(csv_rows, expected_rows) else: self.assertItemsEqual(csv_rows, expected_rows)
def test_xblockcompletion_get_resumen(self): """ test to generate course survey report and then test the report authenticity. """ from lms.djangoapps.courseware.models import StudentModule data = {'format': True, 'course': str(self.course.id), 'base_url':'this_is_a_url'} task_input = {'data': data } module = StudentModule( module_state_key=self.items[0].location, student=self.student, course_id=self.course.id, module_type='problem', state='{"score": {"raw_earned": 0, "raw_possible": 3}, "seed": 1, "attempts":1}') module.save() with patch('lms.djangoapps.instructor_task.tasks_helper.runner._get_current_task'): result = generate( None, None, self.course.id, task_input, 'EOL_Xblock_Completion' ) report_store = ReportStore.from_config(config_name='GRADES_DOWNLOAD') header_row = ",".join(['Titulo', 'Username', 'Email', 'Run', 'Seccion', 'SubSeccion', 'Unidad', 'Intentos', 'Pts Ganados', 'Pts Posibles', 'Url', 'block_id']) student1_row = ",".join([ self.items[0].display_name, self.student.username, self.student.email, '', '1.' + self.chapter.display_name, '1.1.' + self.section.display_name, '1.1.1.' + self.subsection.display_name, '1','0','3' ]) expected_data = [header_row, student1_row] self._verify_csv_file_report(report_store, expected_data)
def upload_csv_to_report_store(rows, csv_name, course_id, timestamp, config_name='GRADES_DOWNLOAD'): """ Upload data as a CSV using ReportStore. Arguments: rows: CSV data in the following format (first column may be a header): [ [row1_colum1, row1_colum2, ...], ... ] csv_name: Name of the resulting CSV course_id: ID of the course Returns: report_name: string - Name of the generated report """ report_store = ReportStore.from_config(config_name) report_name = u"{course_prefix}_{csv_name}_{timestamp_str}.csv".format( course_prefix=course_filename_prefix_generator(course_id), csv_name=csv_name, timestamp_str=timestamp.strftime("%Y-%m-%d-%H%M") ) report_store.store_rows(course_id, report_name, rows) tracker_emit(csv_name) return report_name
def upload_csv_to_report_store(rows, csv_name, course_id, timestamp, config_name='GRADES_DOWNLOAD'): """ Upload data as a CSV using ReportStore. Arguments: rows: CSV data in the following format (first column may be a header): [ [row1_colum1, row1_colum2, ...], ... ] csv_name: Name of the resulting CSV course_id: ID of the course """ report_store = ReportStore.from_config(config_name) report_store.store_rows( course_id, u"{course_prefix}_{csv_name}_{timestamp_str}.csv".format( course_prefix=course_filename_prefix_generator(course_id), csv_name=csv_name, timestamp_str=timestamp.strftime("%Y-%m-%d-%H%M") ), rows ) tracker_emit(csv_name)
def upload_csv_to_report_store(rows, csv_name, course_id, timestamp, config_name='GRADES_DOWNLOAD', parent_dir=''): """ Upload data as a CSV using ReportStore. Arguments: rows: CSV data in the following format (first column may be a header): [ [row1_colum1, row1_colum2, ...], ... ] csv_name: Name of the resulting CSV course_id: ID of the course parent_dor: Name of the directory where the CSV file will be stored Returns: report_name: string - Name of the generated report """ report_store = ReportStore.from_config(config_name) report_name = "{course_prefix}_{csv_name}_{timestamp_str}.csv".format( course_prefix=course_filename_prefix_generator(course_id), csv_name=csv_name, timestamp_str=timestamp.strftime("%Y-%m-%d-%H%M")) report_store.store_rows(course_id, report_name, rows, parent_dir) tracker_emit(csv_name) return report_name
def test_gradeucursos_post_from_instructor_tab_grade_cutoff_not_defined_in_report(self, grade): """ Test gradeucursos post from instructor tab when grade cutoff is not defined """ grade.return_value = None try: from uchileedxlogin.models import EdxLoginUser EdxLoginUser.objects.create(user=self.student, run='09472337K') except ImportError: self.skipTest("import error uchileedxlogin") task_input = { 'grade_type': 'seven_scale', 'course_id': str(self.course.id), 'instructor_tab': True, 'assig_type': 'gradeucursos_total', 'is_resumen': False } with patch('lms.djangoapps.instructor_task.tasks_helper.runner._get_current_task'): result = task_get_data( None, None, self.course.id, task_input, 'EOL_GRADE_UCURSOS' ) report_store = ReportStore.from_config(config_name='GRADES_DOWNLOAD') report_csv_filename = report_store.links_for(self.course.id)[0][0] report_path = report_store.path_to(self.course.id, report_csv_filename) self.assertTrue('_Error_notas_estudiantes_' in report_csv_filename) self.assertTrue('_Error_notas_estudiantes_' in report_path)
def graph_course_forums_usage(request, course_id): """ Generate a d3 graphable csv-string by checking the report store for the clicked_on file """ clicked_text = request.POST.get('clicked_on') course_key = SlashSeparatedCourseKey.from_deprecated_string(course_id) report_store = ReportStore.from_config(config_name='GRADES_DOWNLOAD') graph = None if clicked_text: for name, url in report_store.links_for(course_key): if clicked_text in name and 'course_forums' in name: request = urllib2.Request(url) request.add_header('Accept-encoding', 'gzip') try: url_handle = urllib2.urlopen(request) if url_handle.info().get('Content-Encoding') == 'gzip': file_buffer = StringIO.StringIO(url_handle.read()) url_handle = gzip.GzipFile(fileobj=file_buffer) graph = generate_course_forums_d3(url_handle) except Exception as error: LOG.error( "Error opening graph_course_forums_usage data report: %s", error, ) break if graph: response = JsonResponse({ 'data': graph, 'filename': clicked_text, }) else: response = JsonResponse({ 'data': 'failure', }) return response
def create_report_store(self): """ Create and return a DjangoStorageReportStore using the old S3ReportStore configuration. """ connection = boto.connect_s3() connection.create_bucket(settings.GRADES_DOWNLOAD['BUCKET']) return ReportStore.from_config(config_name='GRADES_DOWNLOAD')
def create_report_store(self): """ Create and return a DjangoStorageReportStore configured to use the local filesystem for storage. """ test_settings = copy.deepcopy(settings.GRADES_DOWNLOAD) test_settings['STORAGE_KWARGS'] = {'location': settings.GRADES_DOWNLOAD['ROOT_PATH']} with override_settings(GRADES_DOWNLOAD=test_settings): return ReportStore.from_config(config_name='GRADES_DOWNLOAD')
def download_url_for_last_report(self): """ Get the URL for the last report, if any """ # Unfortunately this is a bit inefficient due to the ReportStore API if not self.last_export_result or self.last_export_result['error'] is not None: return None from lms.djangoapps.instructor_task.models import ReportStore report_store = ReportStore.from_config(config_name='GRADES_DOWNLOAD') course_key = getattr(self.scope_ids.usage_id, 'course_key', None) return dict(report_store.links_for(course_key)).get(self.last_export_result['report_filename'])
def get_csv_row_with_headers(self): """ Helper function to return list with the column names from the CSV file (the first row) """ report_store = ReportStore.from_config(config_name='GRADES_DOWNLOAD') report_csv_filename = report_store.links_for(self.course.id)[0][0] report_path = report_store.path_to(self.course.id, report_csv_filename) with report_store.storage.open(report_path) as csv_file: rows = unicodecsv.reader(csv_file, encoding='utf-8-sig') return next(rows)
def get_csv_row_with_headers(self): """ Helper function to return list with the column names from the CSV file (the first row) """ report_store = ReportStore.from_config(config_name='GRADES_DOWNLOAD') report_csv_filename = report_store.links_for(self.course.id)[0][0] report_path = report_store.path_to(self.course.id, report_csv_filename) with report_store.storage.open(report_path) as csv_file: rows = unicodecsv.reader(csv_file, encoding='utf-8') return rows.next()
def delete_report_download(request, course_id): """ Delete a downloaded report from the Instructor Dashboard """ course_id = SlashSeparatedCourseKey.from_string(course_id) filename = request.POST.get('filename') report_store = ReportStore.from_config(config_name='GRADES_DOWNLOAD') report_store.delete_file(course_id, filename) message = { 'status': _('The report was successfully deleted!'), } return JsonResponse(message)
def save_archive(self, course_id, filename, timestamp, config_name='GRADES_DOWNLOAD'): report_store = ReportStore.from_config(config_name) zip_file_name = u"{filename}_{course}_{timestamp_str}.zip".format( filename=filename, course=get_valid_filename( unicode("_").join( [course_id.org, course_id.course, course_id.run])), timestamp_str=timestamp.strftime("%Y-%m-%d-%H%M")) report_store.store(course_id, zip_file_name, self.archive.read())
def test_delete_report(self): report_store = ReportStore.from_config(config_name='GRADES_DOWNLOAD') task_input = {'features': []} links = report_store.links_for(self.course.id) self.assertEquals(len(links), 0) with patch('lms.djangoapps.instructor_task.tasks_helper._get_current_task'): upload_students_csv(None, None, self.course.id, task_input, 'calculated') links = report_store.links_for(self.course.id) self.assertEquals(len(links), 1) filename = links[0][0] report_store.delete_file(self.course.id, filename) links = report_store.links_for(self.course.id) self.assertEquals(len(links), 0)
def test_delete_report(self): report_store = ReportStore.from_config(config_name='GRADES_DOWNLOAD') task_input = {'features': []} links = report_store.links_for(self.course.id) self.assertEquals(len(links), 0) with patch('lms.djangoapps.instructor_task.tasks_helper.runner._get_current_task'): upload_students_csv(None, None, self.course.id, task_input, 'calculated') links = report_store.links_for(self.course.id) self.assertEquals(len(links), 1) filename = links[0][0] report_store.delete_file(self.course.id, filename) links = report_store.links_for(self.course.id) self.assertEquals(len(links), 0)
def test_financial_report_overrides(self): """ Test that CUSTOM_DOMAIN from FINANCIAL_REPORTS is used to construct file url. instead of domain defined via AWS_S3_CUSTOM_DOMAIN setting. """ with override_settings(FINANCIAL_REPORTS={ 'STORAGE_TYPE': 's3', 'BUCKET': 'edx-financial-reports', 'CUSTOM_DOMAIN': 'edx-financial-reports.s3.amazonaws.com', 'ROOT_PATH': 'production', }): report_store = ReportStore.from_config(config_name="FINANCIAL_REPORTS") # Make sure CUSTOM_DOMAIN from FINANCIAL_REPORTS is used to construct file url self.assertIn("edx-financial-reports.s3.amazonaws.com", report_store.storage.url(""))
def upload_csv_to_report_store(rows, csv_name, course_id, timestamp, config_name='GRADES_DOWNLOAD'): report_store = ReportStore.from_config(config_name) report_name = u"{course_prefix}_{csv_name}_{timestamp_str}.csv".format( course_prefix=course_filename_prefix_generator(course_id), csv_name=csv_name, timestamp_str=timestamp.strftime("%Y-%m-%d-%H%M")) report_store.store_rows(course_id, report_name, rows) tracker_emit(csv_name) return report_name
def _get_report_urls(self, context): report_store = ReportStore.from_config(config_name='GRADES_DOWNLOAD') files_urls_pairs = report_store.links_for(context.course_id) get_first = lambda iterable: iterable[0] if len(iterable) else None find_by_name = lambda name: get_first( [url for filename, url in files_urls_pairs if name in filename]) report_name = self.REPORT_NAME_TEMPLATE.format(task_id=context.task_id) err_report_name = self.REPORT_NAME_TEMPLATE.format( task_id=context.task_id) csv_url = find_by_name(report_name) csv_err_url = find_by_name(err_report_name) return csv_url, csv_err_url
def create_report_store(self): """ Create and return a DjangoStorageReportStore configured to use S3 for storage. """ test_settings = copy.deepcopy(settings.GRADES_DOWNLOAD) test_settings['STORAGE_CLASS'] = 'storages.backends.s3boto.S3BotoStorage' test_settings['STORAGE_KWARGS'] = { 'bucket': settings.GRADES_DOWNLOAD['BUCKET'], 'location': settings.GRADES_DOWNLOAD['ROOT_PATH'], } with override_settings(GRADES_DOWNLOAD=test_settings): self.mocked_connection.create_bucket(settings.GRADES_DOWNLOAD['STORAGE_KWARGS']['bucket']) return ReportStore.from_config(config_name='GRADES_DOWNLOAD')
def save_csv(self, course_id, filename, cvs_file, timestamp, config_name='GRADES_DOWNLOAD'): report_store = ReportStore.from_config(config_name) csv_filename = u"{filename}_{course}_{timestamp_str}.csv".format( filename=filename, course=get_valid_filename( unicode("_").join( [course_id.org, course_id.course, course_id.run])), timestamp_str=timestamp.strftime("%Y-%m-%d-%H%M")) report_store.store(course_id, csv_filename, cvs_file)
def upload_zip_to_report_store(file, zip_name, course_id, timestamp, config_name='GRADES_DOWNLOAD'): """ Upload given file buffer as a zip file using ReportStore. """ report_store = ReportStore.from_config(config_name) report_name = "{course_prefix}_{zip_name}_{timestamp_str}.zip".format( course_prefix=course_filename_prefix_generator(course_id), zip_name=zip_name, timestamp_str=timestamp.strftime("%Y-%m-%d-%H%M") ) report_store.store(course_id, report_name, file) tracker_emit(zip_name) return report_name
def create_report_store(self): """ Create and return a DjangoStorageReportStore configured to use S3 for storage. """ test_settings = copy.deepcopy(settings.GRADES_DOWNLOAD) test_settings['STORAGE_CLASS'] = 'openedx.core.storage.S3ReportStorage' test_settings['STORAGE_KWARGS'] = { 'bucket': settings.GRADES_DOWNLOAD['BUCKET'], 'location': settings.GRADES_DOWNLOAD['ROOT_PATH'], } with override_settings(GRADES_DOWNLOAD=test_settings): connection = boto.connect_s3() connection.create_bucket(settings.GRADES_DOWNLOAD['STORAGE_KWARGS']['bucket']) return ReportStore.from_config(config_name='GRADES_DOWNLOAD')
def test_xblockcompletion_get_all_data(self, report): """ Test xblockcompletion view all data """ state_1 = {_("Answer ID"): 'answer_id', _("Question"): 'question_text', _("Answer"): 'answer_text', _("Correct Answer") : 'correct_answer_text' } state_2 = {_("Answer ID"): 'answer_id', _("Question"): 'question_text', _("Answer"): 'correct_answer_text', _("Correct Answer") : 'correct_answer_text' } generated_report_data = {self.student.username : [state_1,state_2,state_1]} report.return_value = generated_report_data from lms.djangoapps.courseware.models import StudentModule data = {'format': False, 'course': str(self.course.id), 'base_url':'this_is_a_url'} task_input = {'data': data } module = StudentModule( module_state_key=self.items[0].location, student=self.student, course_id=self.course.id, module_type='problem', state='{"score": {"raw_earned": 1, "raw_possible": 3}, "seed": 1, "attempts": 1}') module.save() with patch('lms.djangoapps.instructor_task.tasks_helper.runner._get_current_task'): result = generate( None, None, self.course.id, task_input, 'EOL_Xblock_Completion' ) report_store = ReportStore.from_config(config_name='GRADES_DOWNLOAD') header_row = ",".join(['Titulo', 'Username', 'Email', 'Run', 'Seccion', 'SubSeccion', 'Unidad', 'Pregunta', 'Respuesta Estudiante', 'Resp. Correcta', 'Intentos', 'Pts Ganados', 'Pts Posibles', 'Pts Total Componente', 'Url', 'block_id']) base_student_row = ",".join([ self.items[0].display_name, self.student.username, self.student.email, '', '1.' + self.chapter.display_name, '1.1.' + self.section.display_name, '1.1.1.' + self.subsection.display_name ]) student_row = base_student_row + ',question_text,answer_text,correct_answer_text,1,0,1.0,3' student_row2 = base_student_row + ',question_text,correct_answer_text,correct_answer_text,1,1.0,1.0,3' expected_data = [header_row, student_row, student_row2, student_row] self._verify_csv_file_report(report_store, expected_data)
def generate(_xmodule_instance_args, _entry_id, course_id, task_input, action_name): """ For a given `course_id`, generate a CSV file containing all student answers to a given problem, and store using a `ReportStore`. """ start_time = time() start_date = datetime.now(UTC) num_reports = 1 task_progress = TaskProgress(action_name, num_reports, start_time) current_step = {'step': 'XblockCompletion - Calculating students answers to problem'} task_progress.update_task_state(extra_meta=current_step) data = task_input.get('data') filter_types = ['problem'] students = XblockCompletionView().get_all_enrolled_users(data['course']) course_structure = get_data_course(data['course']) report_store = ReportStore.from_config('GRADES_DOWNLOAD') csv_name = 'Reporte_de_Preguntas' if data['format']: csv_name = 'Reporte_de_Preguntas_Resumen' report_name = u"{course_prefix}_{csv_name}_{timestamp_str}.csv".format( course_prefix=course_filename_prefix_generator(course_id), csv_name=csv_name, timestamp_str=start_date.strftime("%Y-%m-%d-%H%M") ) output_buffer = ContentFile('') if six.PY2: output_buffer.write(codecs.BOM_UTF8) csvwriter = csv.writer(output_buffer) student_states = XblockCompletionView().get_all_states(data['course'], filter_types) csvwriter = XblockCompletionView()._build_student_data(data, students, course_structure, student_states, filter_types, csvwriter) current_step = {'step': 'XblockCompletion - Uploading CSV'} task_progress.update_task_state(extra_meta=current_step) output_buffer.seek(0) report_store.store(course_id, report_name, output_buffer) current_step = { 'step': 'XblockCompletion - CSV uploaded', 'report_name': report_name, } return task_progress.update_task_state(extra_meta=current_step)
def get_report_info(csv_name, course_id, timestamp, config_name='GRADES_DOWNLOAD'): """ Returns ReportStore and Report Name. Arguments: csv_name: Name of the resulting CSV course_id: ID of the course Returns: report_stroe: ReportStore - Instance of report store report_name: string - Name of the generated report """ report_store = ReportStore.from_config(config_name) report_name = u"{course_prefix}_{csv_name}_{timestamp_str}.csv".format( course_prefix=course_filename_prefix_generator(course_id), csv_name=csv_name, timestamp_str=timestamp.strftime("%Y-%m-%d-%H%M")) return report_store, report_name
def export_csv_data(block_id, course_id): """ Exports student answers to all supported questions to a CSV file. """ src_block = modulestore().get_item(UsageKey.from_string(block_id)) start_timestamp = time.time() course_key = CourseKey.from_string(course_id) filename = src_block.get_filename() report_store = ReportStore.from_config(config_name='GRADES_DOWNLOAD') report_store.store_rows(course_key, filename, src_block.prepare_data()) generation_time_s = time.time() - start_timestamp return { "error": None, "report_filename": filename, "start_timestamp": start_timestamp, "generation_time_s": generation_time_s, }
def _upload_exec_summary_to_store(data_dict, report_name, course_id, generated_at, config_name='FINANCIAL_REPORTS'): """ Upload Executive Summary Html file using ReportStore. Arguments: data_dict: containing executive report data. report_name: Name of the resulting Html File. course_id: ID of the course """ report_store = ReportStore.from_config(config_name) # Use the data dict and html template to generate the output buffer output_buffer = StringIO(render_to_string("instructor/instructor_dashboard_2/executive_summary.html", data_dict)) report_store.store( course_id, u"{course_prefix}_{report_name}_{timestamp_str}.html".format( course_prefix=course_filename_prefix_generator(course_id), report_name=report_name, timestamp_str=generated_at.strftime("%Y-%m-%d-%H%M") ), output_buffer, ) tracker_emit(report_name)
def export_data(course_id, source_block_id_str, block_types, user_ids, match_string): """ Exports student answers to all supported questions to a CSV file. """ start_timestamp = time.time() logger.debug("Beginning data export") try: course_key = CourseKey.from_string(course_id) usage_key = UsageKey.from_string(source_block_id_str) except InvalidKeyError: raise ValueError("Could not find the specified Block ID.") src_block = modulestore().get_item(usage_key) course_key_str = unicode(course_key) type_map = {cls.__name__: cls for cls in [MCQBlock, MRQBlock, RatingBlock, AnswerBlock]} if not block_types: block_types = tuple(type_map.values()) else: block_types = tuple(type_map[class_name] for class_name in block_types) # Build an ordered list of blocks to include in the export blocks_to_include = [] def scan_for_blocks(block): """ Recursively scan the course tree for blocks of interest """ if isinstance(block, block_types): blocks_to_include.append(block) elif block.has_children: for child_id in block.children: try: scan_for_blocks(block.runtime.get_block(child_id)) except ItemNotFoundError: # Blocks may refer to missing children. Don't break in this case. pass scan_for_blocks(src_block) # Define the header row of our CSV: rows = [] rows.append( ["Section", "Subsection", "Unit", "Type", "Question", "Answer", "Username", "User ID", "User E-mail"] ) # Collect results for each block in blocks_to_include for block in blocks_to_include: if not user_ids: results = _extract_data(course_key_str, block, None, match_string) rows += results else: for user_id in user_ids: results = _extract_data(course_key_str, block, user_id, match_string) rows += results # Generate the CSV: filename = u"pb-data-export-{}.csv".format(time.strftime("%Y-%m-%d-%H%M%S", time.gmtime(start_timestamp))) report_store = ReportStore.from_config(config_name='GRADES_DOWNLOAD') report_store.store_rows(course_key, filename, rows) generation_time_s = time.time() - start_timestamp logger.debug("Done data export - took {} seconds".format(generation_time_s)) return { "error": None, "report_filename": filename, "start_timestamp": start_timestamp, "generation_time_s": generation_time_s, "display_data": [] if len(rows) == 1 else rows[1:1001] # Limit to preview of 1000 items }
def create_report_store(self): """ Create and return a DjangoStorageReportStore using the old LocalFSReportStore configuration. """ return ReportStore.from_config(config_name='GRADES_DOWNLOAD')
def export_data(course_id, source_block_id_str, block_types, user_ids, match_string): """ Exports student answers to all MCQ questions to a CSV file. """ start_timestamp = time.time() logger.debug("Beginning data export") try: course_key = CourseKey.from_string(course_id) src_block = modulestore().get_items(course_key, qualifiers={'name': source_block_id_str}, depth=0)[0] except IndexError: raise ValueError("Could not find the specified Block ID.") course_key_str = unicode(course_key) type_map = {cls.__name__: cls for cls in [MCQBlock, RatingBlock, AnswerBlock]} if not block_types: block_types = tuple(type_map.values()) else: block_types = tuple(type_map[class_name] for class_name in block_types) # Build an ordered list of blocks to include in the export blocks_to_include = [] def scan_for_blocks(block): """ Recursively scan the course tree for blocks of interest """ if isinstance(block, block_types): blocks_to_include.append(block) elif block.has_children: for child_id in block.children: try: scan_for_blocks(block.runtime.get_block(child_id)) except ItemNotFoundError: # Blocks may refer to missing children. Don't break in this case. pass scan_for_blocks(src_block) # Define the header row of our CSV: rows = [] rows.append(["Section", "Subsection", "Unit", "Type", "Question", "Answer", "Username"]) # Collect results for each block in blocks_to_include for block in blocks_to_include: if not user_ids: results = _extract_data(course_key_str, block, None, match_string) rows += results else: for user_id in user_ids: results = _extract_data(course_key_str, block, user_id, match_string) rows += results # Generate the CSV: filename = u"pb-data-export-{}.csv".format(time.strftime("%Y-%m-%d-%H%M%S", time.gmtime(start_timestamp))) report_store = ReportStore.from_config(config_name='GRADES_DOWNLOAD') report_store.store_rows(course_key, filename, rows) generation_time_s = time.time() - start_timestamp logger.debug("Done data export - took {} seconds".format(generation_time_s)) return { "error": None, "report_filename": filename, "start_timestamp": start_timestamp, "generation_time_s": generation_time_s, "display_data": [] if len(rows) == 1 else rows[1:1001] # Limit to preview of 1000 items }
def upload_user_grades_csv(_xmodule_instance_args, _entry_id, course_id, _task_input, action_name): # pylint: disable=too-many-statements """ For a given `course_id`, for given usernames generates a grades CSV file, and store using a `ReportStore`. Once created, the files can be accessed by instantiating another `ReportStore` (via `ReportStore.from_config()`) and calling `link_for()` on it. Unenrolled users and unknown usernames are stored in *_err_*.csv This task is very close to the .upload_grades_csv from instructor_tasks.task_helper The difference is that we filter enrolled students against requested usernames and we push info about this into PLP """ start_time = time() start_date = datetime.now(UTC) status_interval = 100 fmt = u'Task: {task_id}, InstructorTask ID: {entry_id}, Course: {course_id}, Input: {task_input}' task_info_string = fmt.format( task_id=_xmodule_instance_args.get('task_id') if _xmodule_instance_args is not None else None, entry_id=_entry_id, course_id=course_id, task_input=_task_input ) TASK_LOG.info(u'%s, Task type: %s, Starting task execution', task_info_string, action_name) extended_kwargs_id = _task_input.get("extended_kwargs_id") extended_kwargs = InstructorTaskExtendedKwargs.get_kwargs_for_id(extended_kwargs_id) usernames = extended_kwargs.get("usernames", None) err_rows = [["id", "username", "error_msg"]] if usernames is None: message = "Error occured during edx task execution: no usersnames in InstructorTaskExtendedKwargs." TASK_LOG.error(u'%s, Task type: %s, ' + message, task_info_string) err_rows.append(["-1", "__", message]) usernames = [] enrolled_students = CourseEnrollment.objects.users_enrolled_in(course_id) enrolled_students = enrolled_students.filter(username__in=usernames) total_enrolled_students = enrolled_students.count() requester_id = _task_input.get("requester_id") task_progress = TaskProgress(action_name, total_enrolled_students, start_time) course = get_course_by_id(course_id) course_is_cohorted = is_course_cohorted(course.id) teams_enabled = course.teams_enabled cohorts_header = ['Cohort Name'] if course_is_cohorted else [] teams_header = ['Team Name'] if teams_enabled else [] experiment_partitions = get_split_user_partitions(course.user_partitions) group_configs_header = [u'Experiment Group ({})'.format(partition.name) for partition in experiment_partitions] certificate_info_header = ['Certificate Eligible', 'Certificate Delivered', 'Certificate Type'] certificate_whitelist = CertificateWhitelist.objects.filter(course_id=course_id, whitelist=True) whitelisted_user_ids = [entry.user_id for entry in certificate_whitelist] # Loop over all our students and build our CSV lists in memory rows = [] current_step = {'step': 'Calculating Grades'} TASK_LOG.info( u'%s, Task type: %s, Current step: %s, Starting grade calculation for total students: %s', task_info_string, action_name, current_step, total_enrolled_students, ) found_students = User.objects.filter(username__in=usernames) # Check invalid usernames if len(found_students)!= len(usernames): found_students_usernames = [x.username for x in found_students] for u in usernames: if u not in found_students_usernames: err_rows.append([-1, u, "invalid_username"]) # Check not enrolled requested students if found_students != enrolled_students: diff = found_students.exclude(id__in=enrolled_students) for u in diff: if u in diff: err_rows.append([u.id, u.username, "enrollment_for_username_not_found"]) total_enrolled_students = enrolled_students.count() student_counter = 0 TASK_LOG.info( u'%s, Task type: %s, Current step: %s, Starting grade calculation for total students: %s', task_info_string, action_name, current_step, total_enrolled_students ) graded_assignments = course.grading.graded_assignments(course_id) grade_header = course.grading.grade_header(graded_assignments) rows.append( ["Student ID", "Email", "Username", "Last Name", "First Name", "Second Name", "Grade", "Grade Percent"] + grade_header + cohorts_header + group_configs_header + teams_header + ['Enrollment Track', 'Verification Status'] + certificate_info_header ) for student, course_grade, err_msg in CourseGradeFactory().iter(course, enrolled_students): # Periodically update task status (this is a cache write) if task_progress.attempted % status_interval == 0: task_progress.update_task_state(extra_meta=current_step) task_progress.attempted += 1 # Now add a log entry after each student is graded to get a sense # of the task's progress student_counter += 1 TASK_LOG.info( u'%s, Task type: %s, Current step: %s, Grade calculation in-progress for students: %s/%s', task_info_string, action_name, current_step, student_counter, total_enrolled_students ) if not course_grade: # An empty course_grade means we failed to grade a student. task_progress.failed += 1 err_rows.append([student.id, student.username, err_msg]) continue # We were able to successfully grade this student for this course. task_progress.succeeded += 1 cohorts_group_name = [] if course_is_cohorted: group = get_cohort(student, course_id, assign=False) cohorts_group_name.append(group.name if group else '') group_configs_group_names = [] for partition in experiment_partitions: group = PartitionService(course_id).get_group(student, partition, assign=False) group_configs_group_names.append(group.name if group else '') team_name = [] if teams_enabled: try: membership = CourseTeamMembership.objects.get(user=student, team__course_id=course_id) team_name.append(membership.team.name) except CourseTeamMembership.DoesNotExist: team_name.append('') enrollment_mode = CourseEnrollment.enrollment_mode_for_user(student, course_id)[0] verification_status = SoftwareSecurePhotoVerification.verification_status_for_user( student, course_id, enrollment_mode ) certificate_info = certificate_info_for_user( student, course_id, course_grade.letter_grade, student.id in whitelisted_user_ids ) second_name = '' try: up = UserProfile.objects.get(user=student) if up.goals: second_name = json.loads(up.goals).get('second_name', '') except ValueError: pass if certificate_info[0] == 'Y': TASK_LOG.info( u'Student is marked eligible_for_certificate' u'(user=%s, course_id=%s, grade_percent=%s gradecutoffs=%s, allow_certificate=%s, is_whitelisted=%s)', student, course_id, course_grade.percent, course.grade_cutoffs, student.profile.allow_certificate, student.id in whitelisted_user_ids ) grade_results = course.grading.grade_results(graded_assignments, course_grade) grade_results = list(chain.from_iterable(grade_results)) rows.append( [student.id, student.email, student.username, student.last_name, student.first_name, second_name, course_grade.percent, course_grade.percent*100] + grade_results + cohorts_group_name + group_configs_group_names + team_name + [enrollment_mode] + [verification_status] + certificate_info ) TASK_LOG.info( u'%s, Task type: %s, Current step: %s, Grade calculation completed for students: %s/%s', task_info_string, action_name, current_step, student_counter, total_enrolled_students ) # By this point, we've got the rows we're going to stuff into our CSV files. current_step = {'step': 'Uploading CSVs'} task_progress.update_task_state(extra_meta=current_step) TASK_LOG.info(u'%s, Task type: %s, Current step: %s', task_info_string, action_name, current_step) # Perform the actual upload custom_grades_download = get_custom_grade_config() report_hash_unique_hash = hex(random.getrandbits(32))[2:] report_name = 'plp_grade_users_report_{}_id_{}'.format(report_hash_unique_hash, requester_id) err_report_name = 'plp_grade_users_report_err_{}_id_{}'.format(report_hash_unique_hash, requester_id) upload_csv_to_report_store(rows, report_name, course_id, start_date, config_name=custom_grades_download) # If there are any error rows (don't count the header), write them out as well has_errors = len(err_rows) > 1 if has_errors: upload_csv_to_report_store(err_rows, err_report_name, course_id, start_date, config_name=custom_grades_download) callback_url = _task_input.get("callback_url", None) if callback_url: report_store = ReportStore.from_config(config_name=custom_grades_download) files_urls_pairs = report_store.links_for(course_id) find_by_name = lambda name: [url for filename, url in files_urls_pairs if name in filename][0] try: csv_url = find_by_name(report_name) csv_err_url = find_by_name(err_report_name) if has_errors else None PlpApiClient().push_grade_api_result(callback_url, csv_url, csv_err_url) except Exception as e: TASK_LOG.error("Failed push to PLP:{}".format(str(e))) # One last update before we close out... TASK_LOG.info(u'%s, Task type: %s, Finalizing grade task', task_info_string, action_name) return task_progress.update_task_state(extra_meta=current_step)