def test_should_validate_datasenders(self):
form_model_mock, project_mock = MagicMock(), MagicMock()
form_model_mock.form_fields = []
form_model_mock.data_senders = ['rep2']
entity_question = PropertyMock(return_value=None)
type(form_model_mock).entity_question = entity_question
form_model_mock.validate_submission.return_value = ([], {})
#with patch("datawinners.project.submission.validator.translate_errors") as translate_errors:
# translate_errors.return_value = ["error_msg"]
validator = SubmissionWorkbookRowValidator(Mock(), form_model_mock)
parsed_rows = [
OrderedDict({'dsid': 'rep3'}),
OrderedDict({
'user_dsid': 'rep4',
'dsid': 'rep4'
})
]
valid_rows, invalid_rows = validator.validate_rows(parsed_rows)
self.assertEqual(len(valid_rows), 0)
self.assertEqual(len(invalid_rows), 2)
self.assertEquals(invalid_rows[0]['errors'], [
u'The Data Sender you are submitting on behalf of cannot submit to this Questionnaire. Add the Data Sender to the Questionnaire.'
])
self.assertEquals(invalid_rows[1]['errors'], [
u'You are not authorized to submit to this Questionnaire. Add yourself as a Data Sender to the Questionnaire.'
])
def test_should_return_valid_rows(self):
form_model_mock = Mock()
form_model_mock.validate_submission.return_value = ([],[])
validator = SubmissionWorkbookRowValidator(Mock(), form_model_mock)
parsed_rows = [("form_code",{}), ("form_code",{})]
valid_rows, invalid_rows = validator.validate_rows(parsed_rows)
self.assertEqual(len(valid_rows), 2)
self.assertEqual(len(invalid_rows), 0)
def test_should_return_valid_rows(self):
form_model_mock = MagicMock(spec=FormModel)
form_model_mock.validate_submission.return_value = ([], [])
validator = SubmissionWorkbookRowValidator(Mock(), form_model_mock)
parsed_rows = [OrderedDict(), OrderedDict()]
valid_rows, invalid_rows = validator.validate_rows(parsed_rows)
self.assertEqual(len(valid_rows), 2)
self.assertEqual(len(invalid_rows), 0)
def test_should_return_invalid_rows(self):
form_model_mock, project_mock = MagicMock(), MagicMock()
form_model_mock.form_fields = []
form_model_mock.validate_submission.return_value = ([],[{"error":"error_msg"}])
validator = SubmissionWorkbookRowValidator(Mock(), form_model_mock, project_mock)
parsed_rows = [("form_code",{}), ("form_code",{})]
valid_rows, invalid_rows = validator.validate_rows(parsed_rows)
self.assertEqual(len(valid_rows), 0)
self.assertEqual(len(invalid_rows), 2)
def test_should_return_valid_rows_for_an_datasender_associated(self):
form_model_mock, project_mock = MagicMock(), MagicMock()
form_model_mock.form_fields = []
form_model_mock.data_senders = ['rep2']
form_model_mock.validate_submission.return_value = ([], {})
validator = SubmissionWorkbookRowValidator(Mock(), form_model_mock)
parsed_rows = [OrderedDict({'dsid': 'rep2', 'user_dsid': 'rep2'})]
valid_rows, invalid_rows = validator.validate_rows(parsed_rows, False)
self.assertEqual(len(valid_rows), 1)
self.assertEqual(len(invalid_rows), 0)
self.assertEquals(
valid_rows[0],
OrderedDict([('dsid', 'rep2'), ('user_dsid', 'rep2')]))
def test_should_return_invalid_rows_for_a_datasender_disassociated(self):
form_model_mock, project_mock = MagicMock(), MagicMock()
form_model_mock.form_fields = []
form_model_mock.data_senders = ['rep2']
form_model_mock.validate_submission.return_value = ([], {})
validator = SubmissionWorkbookRowValidator(Mock(), form_model_mock)
parsed_rows = [OrderedDict({'dsid': 'rep3', 'user_dsid': 'rep3'})]
valid_rows, invalid_rows = validator.validate_rows(parsed_rows, False)
self.assertEqual(len(valid_rows), 0)
self.assertEqual(len(invalid_rows), 1)
self.assertEquals(invalid_rows[0]['errors'], [
'You are not authorized to submit to this Questionnaire. Add yourself as a Data Sender to the Questionnaire.'
])
def test_should_return_invalid_rows_for_disassociated_datasenders_by_an_organization_user(
self):
form_model_mock, project_mock = MagicMock(), MagicMock()
form_model_mock.form_fields = []
form_model_mock.data_senders = ['rep2']
form_model_mock.validate_submission.return_value = ([], {})
validator = SubmissionWorkbookRowValidator(Mock(), form_model_mock)
parsed_rows = [OrderedDict({'dsid': 'rep7', 'user_dsid': 'osi1'})]
valid_rows, invalid_rows = validator.validate_rows(parsed_rows, True)
self.assertEqual(len(valid_rows), 0)
self.assertEqual(len(invalid_rows), 1)
self.assertEquals(invalid_rows[0]['errors'], [
"The Data Sender you are submitting on behalf of cannot submit to this Questionnaire. Add the Data Sender to the Questionnaire."
])
def test_should_return_invalid_rows(self):
form_model_mock, project_mock = MagicMock(), MagicMock()
form_model_mock.form_fields = []
entity_question = PropertyMock(return_value=None)
type(form_model_mock).entity_question = entity_question
form_model_mock.validate_submission.return_value = ([], {
"error":
"error_msg"
})
with patch("datawinners.project.submission.validator.translate_errors"
) as translate_errors:
translate_errors.return_value = ["error_msg"]
validator = SubmissionWorkbookRowValidator(Mock(), form_model_mock)
parsed_rows = [OrderedDict(), OrderedDict()]
valid_rows, invalid_rows = validator.validate_rows(parsed_rows)
self.assertEqual(len(valid_rows), 0)
self.assertEqual(len(invalid_rows), 2)
self.assertEquals(invalid_rows[0]['errors'], ['error_msg'])
self.assertEquals(invalid_rows[1]['errors'], ['error_msg'])
class SubmissionImporter():
def __init__(self, dbm, feed_dbm, user, form_model,
submission_quota_service):
self.dbm = dbm
self.user = user
self.form_model = form_model
self.submission_validator = SubmissionWorkbookRowValidator(
dbm, form_model)
self.submission_persister = SubmissionPersister(
user, dbm, feed_dbm, form_model, submission_quota_service)
def import_submission(self, request):
saved_entries, invalid_row_details, ignored_entries = [], [], []
total_submissions = 0
try:
file_content, parser = self._get_content_and_parser(request)
is_organization_user = is_org_user(self.user)
tabular_data = parser().parse(file_content)
if len(tabular_data) <= 1:
raise ImportValidationError(
gettext("The imported file is empty."))
q_answer_dicts = SubmissionWorkbookMapper(
tabular_data, self.form_model).process()
SubmissionWorkbookValidator(
self.form_model).validate(q_answer_dicts)
user_profile = NGOUserProfile.objects.filter(user=self.user)[0]
#todo add this when default reporter question gets added
self._add_reporter_id_for_datasender(q_answer_dicts, user_profile,
is_organization_user)
valid_rows, invalid_row_details = self.submission_validator.validate_rows(
q_answer_dicts, is_organization_user)
ignored_entries, saved_entries = self.submission_persister.save_submissions(
is_organization_user, user_profile, valid_rows)
total_submissions = len(invalid_row_details) + len(valid_rows)
if ignored_entries:
ignored_row_start = total_submissions - len(
ignored_entries) + 1
message = gettext(
"You have crossed the 1000 Submissions limit for your Basic account. Submissions from row %s have been ignored and were not imported."
) % str(ignored_row_start)
else:
message = gettext(
'%s of %s Submissions imported. Please check below for details.'
) % (len(saved_entries), total_submissions)
except InvalidFileFormatException as e:
message = gettext(
u"We could not import your data ! You are using a document format we canʼt import. Please use the excel (.xlsx) template file!"
)
except ImportValidationError as e:
message = e.message
except Exception as e:
message = gettext(
"Some unexpected error happened. Please check the excel file and import again."
)
logger.exception("Submission import failed")
return SubmissionImportResponse(
saved_entries=saved_entries,
errored_entrie_details=invalid_row_details,
ignored_entries=ignored_entries,
message=message,
total_submissions=total_submissions)
def _get_content_and_parser(self, request):
file_name, file_content = get_filename_and_contents(request)
base_name, extension = os.path.splitext(file_name)
if extension not in ['.xls', '.xlsx']:
raise InvalidFileFormatException()
parser_dict = {'.xls': XlsSubmissionParser, '.xlsx': XlsxParser}
return file_content, parser_dict.get(extension)
def _add_reporter_id_for_datasender(self, parsed_rows, user_profile,
is_organization_user):
for row in parsed_rows:
if is_organization_user:
if row['dsid'] == '':
row.update({'dsid': user_profile.reporter_id})
else:
row.update({"dsid": user_profile.reporter_id})
row.update({'user_dsid': user_profile.reporter_id})