def _get_csv_file_content(csv_file): """ returns appropriate csv file content based on input and output is compatible with python versions """ if (not isinstance(csv_file, str)) and six.PY3: content = csv_file.read() else: content = csv_file if isinstance(content, bytes): csv_content = content.decode('utf-8') else: csv_content = content if six.PY3: return csv_content else: return UniversalNewlineIterator(csv_content)
def cohort_students_and_upload(_xmodule_instance_args, _entry_id, course_id, task_input, action_name): """ Within a given course, cohort students in bulk, then upload the results using a `ReportStore`. """ start_time = time() start_date = datetime.now(UTC) # Iterate through rows to get total assignments for task progress with DefaultStorage().open(task_input['file_name']) as f: total_assignments = 0 for _line in unicodecsv.DictReader(UniversalNewlineIterator(f)): total_assignments += 1 task_progress = TaskProgress(action_name, total_assignments, start_time) current_step = {'step': 'Cohorting Students'} task_progress.update_task_state(extra_meta=current_step) # cohorts_status is a mapping from cohort_name to metadata about # that cohort. The metadata will include information about users # successfully added to the cohort, users not found, Preassigned # users, and a cached reference to the corresponding cohort object # to prevent redundant cohort queries. cohorts_status = {} with DefaultStorage().open(task_input['file_name']) as f: for row in unicodecsv.DictReader(UniversalNewlineIterator(f), encoding='utf-8'): # Try to use the 'email' field to identify the user. If it's not present, use 'username'. username_or_email = row.get('email') or row.get('username') cohort_name = row.get('cohort') or '' task_progress.attempted += 1 if not cohorts_status.get(cohort_name): cohorts_status[cohort_name] = { 'Cohort Name': cohort_name, 'Learners Added': 0, 'Learners Not Found': set(), 'Invalid Email Addresses': set(), 'Preassigned Learners': set() } try: cohorts_status[cohort_name]['cohort'] = CourseUserGroup.objects.get( course_id=course_id, group_type=CourseUserGroup.COHORT, name=cohort_name ) cohorts_status[cohort_name]["Exists"] = True except CourseUserGroup.DoesNotExist: cohorts_status[cohort_name]["Exists"] = False if not cohorts_status[cohort_name]['Exists']: task_progress.failed += 1 continue try: # If add_user_to_cohort successfully adds a user, a user object is returned. # If a user is preassigned to a cohort, no user object is returned (we already have the email address). (user, previous_cohort, preassigned) = add_user_to_cohort(cohorts_status[cohort_name]['cohort'], username_or_email) if preassigned: cohorts_status[cohort_name]['Preassigned Learners'].add(username_or_email) task_progress.preassigned += 1 else: cohorts_status[cohort_name]['Learners Added'] += 1 task_progress.succeeded += 1 except User.DoesNotExist: # Raised when a user with the username could not be found, and the email is not valid cohorts_status[cohort_name]['Learners Not Found'].add(username_or_email) task_progress.failed += 1 except ValidationError: # Raised when a user with the username could not be found, and the email is not valid, # but the entered string contains an "@" # Since there is no way to know if the entered string is an invalid username or an invalid email, # assume that a string with the "@" symbol in it is an attempt at entering an email cohorts_status[cohort_name]['Invalid Email Addresses'].add(username_or_email) task_progress.failed += 1 except ValueError: # Raised when the user is already in the given cohort task_progress.skipped += 1 task_progress.update_task_state(extra_meta=current_step) current_step['step'] = 'Uploading CSV' task_progress.update_task_state(extra_meta=current_step) # Filter the output of `add_users_to_cohorts` in order to upload the result. output_header = ['Cohort Name', 'Exists', 'Learners Added', 'Learners Not Found', 'Invalid Email Addresses', 'Preassigned Learners'] output_rows = [ [ ','.join(status_dict.get(column_name, '')) if (column_name == 'Learners Not Found' or column_name == 'Invalid Email Addresses' or column_name == 'Preassigned Learners') else status_dict[column_name] for column_name in output_header ] for _cohort_name, status_dict in cohorts_status.iteritems() ] output_rows.insert(0, output_header) upload_csv_to_report_store(output_rows, 'cohort_results', course_id, start_date) return task_progress.update_task_state(extra_meta=current_step)
def test_unicode_data(self, buffer_size): self.assertEqual([ thing for thing in UniversalNewlineIterator(StringIO(u'héllø wo®ld'), buffer_size=buffer_size) ], [u'héllø wo®ld'])
def test_empty_file(self, buffer_size): self.assertEqual([ thing for thing in UniversalNewlineIterator(StringIO(u''), buffer_size=buffer_size) ], [])
def test_only_one_line_no_trailing_newline(self, buffer_size): self.assertEqual([ thing for thing in UniversalNewlineIterator(StringIO(u'foo'), buffer_size=buffer_size) ], ['foo'])
def test_carriage_returns_and_line_feeds(self, buffer_size): self.assertEqual([ thing for thing in UniversalNewlineIterator(StringIO(u'foo\r\nbar\r\n'), buffer_size=buffer_size) ], ['foo\n', 'bar\n'])
def bulk_enroll_students_and_upload(_xmodule_instance_args, _entry_id, course_id, task_input, action_name): """ Within a given course, enroll students in bulk, then upload the results using a `ReportStore`. """ start_time = time() start_date = datetime.now(UTC) # Iterate through rows to get total assignments for task progress with DefaultStorage().open(task_input['file_name']) as f: total_assignments = 0 for _line in unicodecsv.DictReader(UniversalNewlineIterator(f)): total_assignments += 1 task_progress = TaskProgress(action_name, total_assignments, start_time) current_step = {'step': 'Bulk Enrollment of Students'} task_progress.update_task_state(extra_meta=current_step) # enrollments_status is a mapping from course enrollments. # The metadata will include information about users successfully enrolled # to the course, users not found, already enrolled user enrollments_status = { 'Course ID': course_id, 'Learners Enrolled': 0, 'Learners Not Found': set(), 'Learners Already Enrolled': set(), 'Learners Failed To Enroll': set(), } with DefaultStorage().open(task_input['file_name']) as f: for row in unicodecsv.DictReader(UniversalNewlineIterator(f), encoding='utf-8'): # Try to use the 'email' field to identify the user. If it's not present, use 'username'. username_or_email = row.get('email') or row.get('username') task_progress.attempted += 1 try: # If enroll_user_to_course successfully enrolls a user or user # is already enrolled, a user object is returned. # If it is not registered, no user object is returned. (user, user_already_enrolled) = enroll_user_to_course( _xmodule_instance_args['request_info'], course_id, username_or_email) if user and user_already_enrolled: enrollments_status['Learners Already Enrolled'].add( username_or_email) task_progress.skipped += 1 elif user and not user_already_enrolled: enrollments_status['Learners Enrolled'] += 1 task_progress.succeeded += 1 else: enrollments_status['Learners Not Found'].add( username_or_email) task_progress.skipped += 1 except: # pylint: disable=bare-except TASK_LOG.exception( u'Exception enrolling user %s in course %s via CSV bulk enrollment', username_or_email, course_id) enrollments_status['Learners Failed To Enroll'].add( username_or_email) task_progress.failed += 1 task_progress.update_task_state(extra_meta=current_step) current_step['step'] = 'Uploading CSV' task_progress.update_task_state(extra_meta=current_step) # Filter the output of `bulk_enroll_users_to_course` in order to upload the result. output_header = [ 'Learners Enrolled', 'Learners Not Found', 'Learners Already Enrolled', 'Learners Failed To Enroll', ] output_rows = [[ ','.join(enrollments_status.get(column_name, '')) if (column_name == 'Learners Not Found' or column_name == 'Learners Already Enrolled' or column_name == 'Learners Failed To Enroll') else enrollments_status[column_name] for column_name in output_header ]] output_rows.insert(0, output_header) upload_csv_to_report_store(output_rows, 'bulk_enrollment_results', course_id, start_date) return task_progress.update_task_state(extra_meta=current_step)
def test_only_one_line(self, buffer_size): self.assertEqual([ thing.decode('utf-8') for thing in UniversalNewlineIterator(StringIO(u'foo\n'), buffer_size=buffer_size) ], ['foo\n'])
def test_no_trailing_newline(self, buffer_size): self.assertEqual([ thing.decode('utf-8') for thing in UniversalNewlineIterator(StringIO(u'foo\nbar'), buffer_size=buffer_size) ], ['foo\n', 'bar'])
def test_carriage_returns(self, buffer_size): self.assertEqual([ thing.decode('utf-8') for thing in UniversalNewlineIterator(StringIO(u'foo\rbar\r'), buffer_size=buffer_size) ], ['foo\n', 'bar\n'])
def cohort_students_and_upload(_xmodule_instance_args, _entry_id, course_id, task_input, action_name): """ Within a given course, cohort students in bulk, then upload the results using a `ReportStore`. """ start_time = time() start_date = datetime.now(UTC) # Iterate through rows to get total assignments for task progress with DefaultStorage().open(task_input['file_name']) as f: total_assignments = 0 for _line in unicodecsv.DictReader(UniversalNewlineIterator(f)): total_assignments += 1 task_progress = TaskProgress(action_name, total_assignments, start_time) current_step = {'step': 'Cohorting Students'} task_progress.update_task_state(extra_meta=current_step) # cohorts_status is a mapping from cohort_name to metadata about # that cohort. The metadata will include information about users # successfully added to the cohort, users not found, and a cached # reference to the corresponding cohort object to prevent # redundant cohort queries. cohorts_status = {} with DefaultStorage().open(task_input['file_name']) as f: for row in unicodecsv.DictReader(UniversalNewlineIterator(f), encoding='utf-8'): # Try to use the 'email' field to identify the user. If it's not present, use 'username'. username_or_email = row.get('email') or row.get('username') cohort_name = row.get('cohort') or '' task_progress.attempted += 1 if not cohorts_status.get(cohort_name): cohorts_status[cohort_name] = { 'Cohort Name': cohort_name, 'Students Added': 0, 'Students Not Found': set() } try: cohorts_status[cohort_name][ 'cohort'] = CourseUserGroup.objects.get( course_id=course_id, group_type=CourseUserGroup.COHORT, name=cohort_name) cohorts_status[cohort_name]["Exists"] = True except CourseUserGroup.DoesNotExist: cohorts_status[cohort_name]["Exists"] = False if not cohorts_status[cohort_name]['Exists']: task_progress.failed += 1 continue try: with transaction.commit_on_success(): add_user_to_cohort(cohorts_status[cohort_name]['cohort'], username_or_email) cohorts_status[cohort_name]['Students Added'] += 1 task_progress.succeeded += 1 except User.DoesNotExist: cohorts_status[cohort_name]['Students Not Found'].add( username_or_email) task_progress.failed += 1 except ValueError: # Raised when the user is already in the given cohort task_progress.skipped += 1 task_progress.update_task_state(extra_meta=current_step) current_step['step'] = 'Uploading CSV' task_progress.update_task_state(extra_meta=current_step) # Filter the output of `add_users_to_cohorts` in order to upload the result. output_header = [ 'Cohort Name', 'Exists', 'Students Added', 'Students Not Found' ] output_rows = [[ ','.join(status_dict.get(column_name, '')) if column_name == 'Students Not Found' else status_dict[column_name] for column_name in output_header ] for _cohort_name, status_dict in cohorts_status.iteritems()] output_rows.insert(0, output_header) upload_csv_to_report_store(output_rows, 'cohort_results', course_id, start_date) return task_progress.update_task_state(extra_meta=current_step)