def _get_csv_file_content(csv_file):
"""
returns appropriate csv file content based on input and output is
compatible with python versions
"""
if (not isinstance(csv_file, str)) and six.PY3:
content = csv_file.read()
else:
content = csv_file
if isinstance(content, bytes):
csv_content = content.decode('utf-8')
else:
csv_content = content
if six.PY3:
return csv_content
else:
return UniversalNewlineIterator(csv_content)
def cohort_students_and_upload(_xmodule_instance_args, _entry_id, course_id, task_input, action_name):
"""
Within a given course, cohort students in bulk, then upload the results
using a `ReportStore`.
"""
start_time = time()
start_date = datetime.now(UTC)
# Iterate through rows to get total assignments for task progress
with DefaultStorage().open(task_input['file_name']) as f:
total_assignments = 0
for _line in unicodecsv.DictReader(UniversalNewlineIterator(f)):
total_assignments += 1
task_progress = TaskProgress(action_name, total_assignments, start_time)
current_step = {'step': 'Cohorting Students'}
task_progress.update_task_state(extra_meta=current_step)
# cohorts_status is a mapping from cohort_name to metadata about
# that cohort. The metadata will include information about users
# successfully added to the cohort, users not found, Preassigned
# users, and a cached reference to the corresponding cohort object
# to prevent redundant cohort queries.
cohorts_status = {}
with DefaultStorage().open(task_input['file_name']) as f:
for row in unicodecsv.DictReader(UniversalNewlineIterator(f), encoding='utf-8'):
# Try to use the 'email' field to identify the user. If it's not present, use 'username'.
username_or_email = row.get('email') or row.get('username')
cohort_name = row.get('cohort') or ''
task_progress.attempted += 1
if not cohorts_status.get(cohort_name):
cohorts_status[cohort_name] = {
'Cohort Name': cohort_name,
'Learners Added': 0,
'Learners Not Found': set(),
'Invalid Email Addresses': set(),
'Preassigned Learners': set()
}
try:
cohorts_status[cohort_name]['cohort'] = CourseUserGroup.objects.get(
course_id=course_id,
group_type=CourseUserGroup.COHORT,
name=cohort_name
)
cohorts_status[cohort_name]["Exists"] = True
except CourseUserGroup.DoesNotExist:
cohorts_status[cohort_name]["Exists"] = False
if not cohorts_status[cohort_name]['Exists']:
task_progress.failed += 1
continue
try:
# If add_user_to_cohort successfully adds a user, a user object is returned.
# If a user is preassigned to a cohort, no user object is returned (we already have the email address).
(user, previous_cohort, preassigned) = add_user_to_cohort(cohorts_status[cohort_name]['cohort'], username_or_email)
if preassigned:
cohorts_status[cohort_name]['Preassigned Learners'].add(username_or_email)
task_progress.preassigned += 1
else:
cohorts_status[cohort_name]['Learners Added'] += 1
task_progress.succeeded += 1
except User.DoesNotExist:
# Raised when a user with the username could not be found, and the email is not valid
cohorts_status[cohort_name]['Learners Not Found'].add(username_or_email)
task_progress.failed += 1
except ValidationError:
# Raised when a user with the username could not be found, and the email is not valid,
# but the entered string contains an "@"
# Since there is no way to know if the entered string is an invalid username or an invalid email,
# assume that a string with the "@" symbol in it is an attempt at entering an email
cohorts_status[cohort_name]['Invalid Email Addresses'].add(username_or_email)
task_progress.failed += 1
except ValueError:
# Raised when the user is already in the given cohort
task_progress.skipped += 1
task_progress.update_task_state(extra_meta=current_step)
current_step['step'] = 'Uploading CSV'
task_progress.update_task_state(extra_meta=current_step)
# Filter the output of `add_users_to_cohorts` in order to upload the result.
output_header = ['Cohort Name', 'Exists', 'Learners Added', 'Learners Not Found', 'Invalid Email Addresses', 'Preassigned Learners']
output_rows = [
[
','.join(status_dict.get(column_name, '')) if (column_name == 'Learners Not Found'
or column_name == 'Invalid Email Addresses'
or column_name == 'Preassigned Learners')
else status_dict[column_name]
for column_name in output_header
]
for _cohort_name, status_dict in cohorts_status.iteritems()
]
output_rows.insert(0, output_header)
upload_csv_to_report_store(output_rows, 'cohort_results', course_id, start_date)
return task_progress.update_task_state(extra_meta=current_step)
def test_unicode_data(self, buffer_size):
self.assertEqual([
thing
for thing in UniversalNewlineIterator(StringIO(u'héllø wo®ld'),
buffer_size=buffer_size)
], [u'héllø wo®ld'])
def test_empty_file(self, buffer_size):
self.assertEqual([
thing
for thing in UniversalNewlineIterator(StringIO(u''),
buffer_size=buffer_size)
], [])
def test_only_one_line_no_trailing_newline(self, buffer_size):
self.assertEqual([
thing
for thing in UniversalNewlineIterator(StringIO(u'foo'),
buffer_size=buffer_size)
], ['foo'])
def test_carriage_returns_and_line_feeds(self, buffer_size):
self.assertEqual([
thing
for thing in UniversalNewlineIterator(StringIO(u'foo\r\nbar\r\n'),
buffer_size=buffer_size)
], ['foo\n', 'bar\n'])
def bulk_enroll_students_and_upload(_xmodule_instance_args, _entry_id,
course_id, task_input, action_name):
"""
Within a given course, enroll students in bulk, then upload the results
using a `ReportStore`.
"""
start_time = time()
start_date = datetime.now(UTC)
# Iterate through rows to get total assignments for task progress
with DefaultStorage().open(task_input['file_name']) as f:
total_assignments = 0
for _line in unicodecsv.DictReader(UniversalNewlineIterator(f)):
total_assignments += 1
task_progress = TaskProgress(action_name, total_assignments, start_time)
current_step = {'step': 'Bulk Enrollment of Students'}
task_progress.update_task_state(extra_meta=current_step)
# enrollments_status is a mapping from course enrollments.
# The metadata will include information about users successfully enrolled
# to the course, users not found, already enrolled user
enrollments_status = {
'Course ID': course_id,
'Learners Enrolled': 0,
'Learners Not Found': set(),
'Learners Already Enrolled': set(),
'Learners Failed To Enroll': set(),
}
with DefaultStorage().open(task_input['file_name']) as f:
for row in unicodecsv.DictReader(UniversalNewlineIterator(f),
encoding='utf-8'):
# Try to use the 'email' field to identify the user. If it's not present, use 'username'.
username_or_email = row.get('email') or row.get('username')
task_progress.attempted += 1
try:
# If enroll_user_to_course successfully enrolls a user or user
# is already enrolled, a user object is returned.
# If it is not registered, no user object is returned.
(user, user_already_enrolled) = enroll_user_to_course(
_xmodule_instance_args['request_info'], course_id,
username_or_email)
if user and user_already_enrolled:
enrollments_status['Learners Already Enrolled'].add(
username_or_email)
task_progress.skipped += 1
elif user and not user_already_enrolled:
enrollments_status['Learners Enrolled'] += 1
task_progress.succeeded += 1
else:
enrollments_status['Learners Not Found'].add(
username_or_email)
task_progress.skipped += 1
except: # pylint: disable=bare-except
TASK_LOG.exception(
u'Exception enrolling user %s in course %s via CSV bulk enrollment',
username_or_email, course_id)
enrollments_status['Learners Failed To Enroll'].add(
username_or_email)
task_progress.failed += 1
task_progress.update_task_state(extra_meta=current_step)
current_step['step'] = 'Uploading CSV'
task_progress.update_task_state(extra_meta=current_step)
# Filter the output of `bulk_enroll_users_to_course` in order to upload the result.
output_header = [
'Learners Enrolled',
'Learners Not Found',
'Learners Already Enrolled',
'Learners Failed To Enroll',
]
output_rows = [[
','.join(enrollments_status.get(column_name, '')) if
(column_name == 'Learners Not Found'
or column_name == 'Learners Already Enrolled' or column_name
== 'Learners Failed To Enroll') else enrollments_status[column_name]
for column_name in output_header
]]
output_rows.insert(0, output_header)
upload_csv_to_report_store(output_rows, 'bulk_enrollment_results',
course_id, start_date)
return task_progress.update_task_state(extra_meta=current_step)
def test_only_one_line(self, buffer_size):
self.assertEqual([
thing.decode('utf-8')
for thing in UniversalNewlineIterator(StringIO(u'foo\n'),
buffer_size=buffer_size)
], ['foo\n'])
def test_no_trailing_newline(self, buffer_size):
self.assertEqual([
thing.decode('utf-8')
for thing in UniversalNewlineIterator(StringIO(u'foo\nbar'),
buffer_size=buffer_size)
], ['foo\n', 'bar'])
def test_carriage_returns(self, buffer_size):
self.assertEqual([
thing.decode('utf-8')
for thing in UniversalNewlineIterator(StringIO(u'foo\rbar\r'),
buffer_size=buffer_size)
], ['foo\n', 'bar\n'])
def cohort_students_and_upload(_xmodule_instance_args, _entry_id, course_id,
task_input, action_name):
"""
Within a given course, cohort students in bulk, then upload the results
using a `ReportStore`.
"""
start_time = time()
start_date = datetime.now(UTC)
# Iterate through rows to get total assignments for task progress
with DefaultStorage().open(task_input['file_name']) as f:
total_assignments = 0
for _line in unicodecsv.DictReader(UniversalNewlineIterator(f)):
total_assignments += 1
task_progress = TaskProgress(action_name, total_assignments, start_time)
current_step = {'step': 'Cohorting Students'}
task_progress.update_task_state(extra_meta=current_step)
# cohorts_status is a mapping from cohort_name to metadata about
# that cohort. The metadata will include information about users
# successfully added to the cohort, users not found, and a cached
# reference to the corresponding cohort object to prevent
# redundant cohort queries.
cohorts_status = {}
with DefaultStorage().open(task_input['file_name']) as f:
for row in unicodecsv.DictReader(UniversalNewlineIterator(f),
encoding='utf-8'):
# Try to use the 'email' field to identify the user. If it's not present, use 'username'.
username_or_email = row.get('email') or row.get('username')
cohort_name = row.get('cohort') or ''
task_progress.attempted += 1
if not cohorts_status.get(cohort_name):
cohorts_status[cohort_name] = {
'Cohort Name': cohort_name,
'Students Added': 0,
'Students Not Found': set()
}
try:
cohorts_status[cohort_name][
'cohort'] = CourseUserGroup.objects.get(
course_id=course_id,
group_type=CourseUserGroup.COHORT,
name=cohort_name)
cohorts_status[cohort_name]["Exists"] = True
except CourseUserGroup.DoesNotExist:
cohorts_status[cohort_name]["Exists"] = False
if not cohorts_status[cohort_name]['Exists']:
task_progress.failed += 1
continue
try:
with transaction.commit_on_success():
add_user_to_cohort(cohorts_status[cohort_name]['cohort'],
username_or_email)
cohorts_status[cohort_name]['Students Added'] += 1
task_progress.succeeded += 1
except User.DoesNotExist:
cohorts_status[cohort_name]['Students Not Found'].add(
username_or_email)
task_progress.failed += 1
except ValueError:
# Raised when the user is already in the given cohort
task_progress.skipped += 1
task_progress.update_task_state(extra_meta=current_step)
current_step['step'] = 'Uploading CSV'
task_progress.update_task_state(extra_meta=current_step)
# Filter the output of `add_users_to_cohorts` in order to upload the result.
output_header = [
'Cohort Name', 'Exists', 'Students Added', 'Students Not Found'
]
output_rows = [[
','.join(status_dict.get(column_name, ''))
if column_name == 'Students Not Found' else status_dict[column_name]
for column_name in output_header
] for _cohort_name, status_dict in cohorts_status.iteritems()]
output_rows.insert(0, output_header)
upload_csv_to_report_store(output_rows, 'cohort_results', course_id,
start_date)
return task_progress.update_task_state(extra_meta=current_step)
