def test_parse_csv_normal(self): header = ("key1", "key2") contents = (("QWERTY", "UIOP"), ("ASDFGH", "JKL:")) with MakeCsvStreamContextManager(header, contents) as stream: expected_result = self._make_expected_result(header, contents) actual_result = list(parse_csv(stream)) assert actual_result == expected_result
def test_parse_csv_check_columns_normal(self): header = ("Name", "MainCaliber") contents = (("Bismarck", "4 x 2 - 380"), ("Yamato", "3 x 3 - 460")) with MakeCsvStreamContextManager(header, contents) as stream: expected_result = self._make_expected_result(header, contents) actual_result = list( parse_csv(stream, expected_columns=set(header))) assert actual_result == expected_result
def test_parse_csv_check_with_wrong_encoding(self): header = ("email",) contents = ( "*****@*****.**", "*****@*****.**", ) with MakeCsvStreamContextManager(header, contents, 'utf-16') as stream: expected_error_message = ValidationMessages.INVALID_ENCODING with raises(ValidationError, match=expected_error_message): list(parse_csv(stream, expected_columns={"email"}))
def test_parse_csv_check_columns_mismatch(self): header = ("Name", "MainCaliber", "Std Displacement") expected_columns = {"Name", "Email"} contents = (("Bismarck", "4 x 2 - 380", "41700"), ("Yamato", "3 x 3 - 460", "63200")) with MakeCsvStreamContextManager(header, contents) as stream: expected_error_message = ValidationMessages.MISSING_EXPECTED_COLUMNS.format( expected_columns=", ".join(expected_columns), actual_columns=", ".join(header)) with raises(ValidationError, message=expected_error_message): list(parse_csv(stream, expected_columns={"Name", "Email"}))
def _handle_bulk_upload(cls, enterprise_customer, manage_learners_form, request, email_list=None): """ Bulk link users by email. Arguments: enterprise_customer (EnterpriseCustomer): learners will be linked to this Enterprise Customer instance manage_learners_form (ManageLearnersForm): bound ManageLearners form instance request (django.http.request.HttpRequest): HTTP Request instance email_list (iterable): A list of pre-processed email addresses to handle using the form """ errors = [] emails = set() course_id_with_emails = {} already_linked_emails = [] duplicate_emails = [] csv_file = manage_learners_form.cleaned_data[ ManageLearnersForm.Fields.BULK_UPLOAD] if email_list: parsed_csv = [{ ManageLearnersForm.CsvColumns.EMAIL: email } for email in email_list] else: parsed_csv = parse_csv( csv_file, expected_columns={ManageLearnersForm.CsvColumns.EMAIL}) try: for index, row in enumerate(parsed_csv): email = row[ManageLearnersForm.CsvColumns.EMAIL] course_id = row.get(ManageLearnersForm.CsvColumns.COURSE_ID, None) # optional column course_details = None try: already_linked = validate_email_to_link( email, ignore_existing=True) if course_id: course_details = validate_course_exists_for_enterprise( enterprise_customer, course_id) except ValidationError as exc: message = _("Error at line {line}: {message}\n").format( line=index + 1, message=exc) errors.append(message) else: if already_linked: already_linked_emails.append( (email, already_linked.enterprise_customer)) elif email in emails: duplicate_emails.append(email) else: emails.add(email) # course column exists for row, is a valid course id, and exists in the enterprise's catalog(s). if course_details: if course_details[ 'course_id'] not in course_id_with_emails: course_id_with_emails[ course_details['course_id']] = {email} else: course_id_with_emails[ course_details['course_id']].add(email) except ValidationError as exc: errors.append(exc) if errors: cls._handle_bulk_upload_errors( cls, manage_learners_form=manage_learners_form, errors=errors) # There were validation errors, so prevent any further action. return [], {} # There were no errors. Now do the actual linking: for email in emails: EnterpriseCustomerUser.objects.link_user(enterprise_customer, email) # Process the bulk uploaded data: processable_emails = cls._process_bulk_upload_data( cls, request=request, enterprise_customer=enterprise_customer, emails=emails, already_linked_emails=already_linked_emails, duplicate_emails=duplicate_emails, ) return processable_emails, course_id_with_emails
def _handle_bulk_upload(cls, enterprise_customer, manage_learners_form, request): """ Bulk link users by email. Arguments: enterprise_customer (EnterpriseCustomer): learners will be linked to this Enterprise Customer instance manage_learners_form (ManageLearnersForm): bound ManageLearners form instance request (django.http.request.HttpRequest): HTTP Request instance """ errors = [] emails = set() already_linked_emails = [] duplicate_emails = [] csv_file = manage_learners_form.cleaned_data[ ManageLearnersForm.Fields.BULK_UPLOAD] try: parsed_csv = parse_csv( csv_file, expected_columns={ManageLearnersForm.CsvColumns.EMAIL}) for index, row in enumerate(parsed_csv): email = row[ManageLearnersForm.CsvColumns.EMAIL] try: already_linked = validate_email_to_link( email, ignore_existing=True) except ValidationError as exc: message = _("Error at line {line}: {message}\n").format( line=index + 1, message=exc.message) errors.append(message) else: if already_linked: already_linked_emails.append(email) elif email in emails: duplicate_emails.append(email) else: emails.add(email) except ValidationError as exc: errors.append(exc.message) if errors: manage_learners_form.add_error( ManageLearnersForm.Fields.GENERAL_ERRORS, ValidationMessages.BULK_LINK_FAILED) for error in errors: manage_learners_form.add_error( ManageLearnersForm.Fields.BULK_UPLOAD, error) return # There were no errors. Now do the actual linking: for email in emails: EnterpriseCustomerUser.objects.link_user(enterprise_customer, email) # Report what happened: count = len(emails) messages.success( request, ungettext( "{count} new user was linked to {enterprise_customer_name}.", "{count} new users were linked to {enterprise_customer_name}.", count).format( count=count, enterprise_customer_name=enterprise_customer.name)) if already_linked_emails: messages.warning( request, _("Some users were already linked to this Enterprise Customer: {list_of_emails}" ).format(list_of_emails=", ".join(already_linked_emails))) if duplicate_emails: messages.warning( request, _("Some duplicate emails in the CSV were ignored: {list_of_emails}" ).format(list_of_emails=", ".join(duplicate_emails))) return list(emails) + already_linked_emails
def _handle_bulk_upload(cls, enterprise_customer, manage_learners_form, request, email_list=None): """ Bulk link users by email. Arguments: enterprise_customer (EnterpriseCustomer): learners will be linked to this Enterprise Customer instance manage_learners_form (ManageLearnersForm): bound ManageLearners form instance request (django.http.request.HttpRequest): HTTP Request instance email_list (iterable): A list of pre-processed email addresses to handle using the form """ errors = [] emails = set() already_linked_emails = [] duplicate_emails = [] csv_file = manage_learners_form.cleaned_data[ ManageLearnersForm.Fields.BULK_UPLOAD] if email_list: parsed_csv = [{ ManageLearnersForm.CsvColumns.EMAIL: email } for email in email_list] else: parsed_csv = parse_csv( csv_file, expected_columns={ManageLearnersForm.CsvColumns.EMAIL}) try: for index, row in enumerate(parsed_csv): email = row[ManageLearnersForm.CsvColumns.EMAIL] try: already_linked = validate_email_to_link( email, ignore_existing=True) except ValidationError as exc: message = _("Error at line {line}: {message}\n").format( line=index + 1, message=exc) errors.append(message) else: if already_linked: already_linked_emails.append( (email, already_linked.enterprise_customer)) elif email in emails: duplicate_emails.append(email) else: emails.add(email) except ValidationError as exc: errors.append(exc) if errors: manage_learners_form.add_error( ManageLearnersForm.Fields.GENERAL_ERRORS, ValidationMessages.BULK_LINK_FAILED) for error in errors: manage_learners_form.add_error( ManageLearnersForm.Fields.BULK_UPLOAD, error) return # There were no errors. Now do the actual linking: for email in emails: EnterpriseCustomerUser.objects.link_user(enterprise_customer, email) # Report what happened: count = len(emails) messages.success( request, ungettext( "{count} new learner was added to {enterprise_customer_name}.", "{count} new learners were added to {enterprise_customer_name}.", count).format( count=count, enterprise_customer_name=enterprise_customer.name)) this_customer_linked_emails = [ email for email, customer in already_linked_emails if customer == enterprise_customer ] other_customer_linked_emails = [ email for email, __ in already_linked_emails if email not in this_customer_linked_emails ] if this_customer_linked_emails: messages.warning( request, _("The following learners were already associated with this Enterprise " "Customer: {list_of_emails}").format( list_of_emails=", ".join(this_customer_linked_emails))) if other_customer_linked_emails: messages.warning( request, _("The following learners are already associated with " "another Enterprise Customer. These learners were not " "added to {enterprise_customer_name}: {list_of_emails}"). format( enterprise_customer_name=enterprise_customer.name, list_of_emails=", ".join(other_customer_linked_emails), )) if duplicate_emails: messages.warning( request, _("The following duplicate email addresses were not added: " "{list_of_emails}").format( list_of_emails=", ".join(duplicate_emails))) # Build a list of all the emails that we can act on further; that is, # emails that we either linked to this customer, or that were linked already. all_processable_emails = list(emails) + this_customer_linked_emails return all_processable_emails