def test_should_replace_answer_option_values_with_options_text_when_answer_type_is_changed_from_single_select_choice_field( self): survey_response_doc = SurveyResponseDocument(values={ 'q1': 'a', }) survey_response = SurveyResponse(Mock()) survey_response._doc = survey_response_doc choice_field = SelectField(name='question one', code='q1', label='question one', options=[("one", "a"), ("two", "b"), ("three", "c"), ("four", "d")], single_select_flag=True) text_field = TextField(name='question one', code='q1', label='question one') questionnaire_form_model = Mock(spec=FormModel) questionnaire_form_model.form_code = 'test_form_code' questionnaire_form_model.fields = [text_field] questionnaire_form_model.get_field_by_code_and_rev.return_value = choice_field request_dict = construct_request_dict(survey_response, questionnaire_form_model, 'id') expected_dict = { 'q1': 'one', 'form_code': 'test_form_code', 'dsid': 'id' } self.assertEqual(request_dict, expected_dict)
def test_should_update_the_associated_submission_when_question_code_is_updated( self): update_dict = { "database_name": "database_name", "old_form_code": "old_form_code", "new_form_code": "new_form_code", "new_revision": "new_revision" } with patch("datawinners.project.wizard_view.get_db_manager" ) as get_db_manager: managerMock = Mock(spec=DatabaseManager) get_db_manager.return_value = managerMock managerMock._save_documents.return_value = [] with patch( "datawinners.project.wizard_view.survey_responses_by_form_code" ) as survey_responses_by_form_code: mock_document1 = SurveyResponseDocument() mock_document2 = SurveyResponseDocument() survey_responses_mock = [ (SurveyResponse.new_from_doc(dbm=None, doc=mock_document1)), (SurveyResponse.new_from_doc(dbm=None, doc=mock_document2)) ] survey_responses_by_form_code.return_value = survey_responses_mock update_submissions_for_form_code_change( managerMock, 'new_form_code', 'old_form_code') managerMock._save_documents.assert_called_with( [mock_document1, mock_document2]) self.assertEquals(mock_document1.form_code, "new_form_code") self.assertEquals(mock_document2.form_code, "new_form_code")
def test_set_answers_uses_case_insensitive_key_value(self): values = {'eid': 'cli001'} survey_response = SurveyResponse(Mock(), transport_info=TransportInfo('web', '*****@*****.**', 'destination'), values=values) survey_response.entity_question_code = 'EID' survey_response.set_answers(values) self.assertEquals(1, len(values))
def test_should_not_update_submissions_no_field_got_deleted(self): dbm = Mock(spec=DatabaseManager) form_code = 'code' deleted_question_codes = [] with patch( "datawinners.project.wizard_view.survey_responses_by_form_model_id" ) as survey_responses_by_form_model_id: mock_document1 = SurveyResponseDocument(values={ 'field_code': 'answer1', 'another_code': 'answer2' }) mock_document2 = SurveyResponseDocument(values={ 'field_code': 'answer3', 'another_code': 'answer4' }) survey_responses_mock = [ (SurveyResponse.new_from_doc(dbm=None, doc=mock_document1)), (SurveyResponse.new_from_doc(dbm=None, doc=mock_document2)) ] survey_responses_by_form_model_id.return_value = survey_responses_mock remove_deleted_questions_from_submissions(dbm, form_code, deleted_question_codes) self.assertEquals(mock_document1.values, { 'field_code': 'answer1', 'another_code': 'answer2' }) self.assertEquals(mock_document2.values, { 'field_code': 'answer3', 'another_code': 'answer4' })
def test_should_get_static_info_from_submission(self): with patch("datawinners.project.views.submission_views.get_data_sender") as get_data_sender: survey_response_document = SurveyResponseDocument(channel='web', status=False, error_message="Some Error in submission") get_data_sender.return_value = ('Psub', 'rep2') submission_date = datetime(2012, 02, 20, 12, 15, 44) survey_response_document.submitted_on = submission_date survey_response_document.created = datetime(2012, 02, 20, 12, 15, 50) survey_response = SurveyResponse(Mock()) survey_response._doc = survey_response_document project = Mock() project.data_senders = ["rep2"] organization_mock = Mock() organization_mock.org_id = "TEST1234" # with patch("datawinners.project.views.submission_views.get_organization_from_manager") as get_ngo_from_manager_mock: # get_ngo_from_manager_mock.return_value = organization_mock static_info = build_static_info_context(Mock(), survey_response, questionnaire_form_model=project) expected_values = OrderedDict({'static_content': { 'Data Sender': ('Psub', 'rep2'), 'Source': u'Web', 'Submission Date': submission_date.strftime(SUBMISSION_DATE_FORMAT_FOR_SUBMISSION)}}) expected_values.update({'is_edit': True}) expected_values.update({'status': u'Error'}) self.assertEqual(expected_values, static_info)
def test_should_get_static_info_from_submission(self): with patch("datawinners.project.views.submission_views.get_data_sender" ) as get_data_sender: survey_response_document = SurveyResponseDocument( channel='web', status=False, error_message="Some Error in submission") get_data_sender.return_value = ('Psub', 'rep2', '*****@*****.**') submission_date = datetime(2012, 02, 20, 12, 15, 44) survey_response_document.submitted_on = submission_date survey_response_document.created = datetime( 2012, 02, 20, 12, 15, 50) survey_response = SurveyResponse(Mock()) survey_response._doc = survey_response_document static_info = build_static_info_context(Mock(), survey_response) expected_values = OrderedDict({ 'static_content': { 'Data Sender': ('Psub', 'rep2', '*****@*****.**'), 'Source': u'Web', 'Submission Date': submission_date.strftime( SUBMISSION_DATE_FORMAT_FOR_SUBMISSION) } }) expected_values.update({'is_edit': True}) expected_values.update({'status': u'Error'}) self.assertEqual(expected_values, static_info)
def test_should_create_request_dict_with_older_survey_response(self): survey_response_doc = SurveyResponseDocument(values={ 'q1': 23, 'q2': 'sometext', 'q3': 'ab' }) survey_response = SurveyResponse(Mock()) survey_response._doc = survey_response_doc int_field = IntegerField(name='question one', code='q1', label='question one') text_field = TextField(name='question two', code='q2', label='question two') choice_field = SelectField(name='question three', code='q4', label='question three', options=[("one", "a"), ("two", "b"), ("three", "c"), ("four", "d")], single_select_flag=False) questionnaire_form_model = Mock(spec=FormModel) questionnaire_form_model.form_code = 'test_form_code' questionnaire_form_model.fields = [int_field, text_field, choice_field] request_dict = construct_request_dict(survey_response, questionnaire_form_model, 'id') expected_dict = { 'q1': 23, 'q2': 'sometext', 'q4': None, 'form_code': 'test_form_code', 'dsid': 'id' } self.assertEqual(request_dict, expected_dict)
def test_convert_survey_response_values_to_dict_format(self): survey_response_doc = SurveyResponseDocument( values={ 'q1': '23', 'q2': 'sometext', 'q3': 'a', 'geo': '2.34,5.64', 'date': '12.12.2012' }) survey_response = SurveyResponse(Mock()) survey_response._doc = survey_response_doc int_field = IntegerField(name='question one', code='q1', label='question one', ddtype=Mock(spec=DataDictType)) text_field = TextField(name='question two', code='q2', label='question two', ddtype=Mock(spec=DataDictType)) single_choice_field = SelectField(name='question three', code='q3', label='question three', options=[("one", "a"), ("two", "b"), ("three", "c"), ("four", "d")], single_select_flag=True, ddtype=Mock(spec=DataDictType)) geo_field = GeoCodeField(name='geo', code='GEO', label='geo', ddtype=Mock()) date_field = DateField(name='date', code='DATE', label='date', date_format='dd.mm.yyyy', ddtype=Mock()) questionnaire_form_model = Mock(spec=FormModel) questionnaire_form_model.form_code = 'test_form_code' questionnaire_form_model.fields = [ int_field, text_field, single_choice_field, geo_field, date_field ] request_dict = construct_request_dict(survey_response, questionnaire_form_model) expected_dict = OrderedDict({ 'q1': '23', 'q2': 'sometext', 'q3': 'a', 'GEO': '2.34,5.64', 'DATE': '12.12.2012', 'form_code': 'test_form_code' }) self.assertEqual(expected_dict, request_dict)
def test_difference_between_two_survey_response_with_with_deleted_question_in_new_questionnaire(self): old_survey_response = SurveyResponse(Mock(), values={'q1': 'ans5', 'q2': 'ans6', 'q3': 'ans3'}, transport_info=TransportInfo('web', 'web', 'web')) old_survey_response._doc.submitted_on = datetime.date(2012, 02, 27) new_survey_response = SurveyResponse(Mock(), values={'q1': 'ans1', 'q2': 'ans2'}, transport_info=TransportInfo('web', 'web', 'web')) new_survey_response._doc.status = True result_diff = new_survey_response.differs_from(old_survey_response) expected_result_diff = SurveyResponseDifference(old_survey_response.submitted_on, True) expected_result_diff.changed_answers = {'q1': {'old': 'ans5', 'new': 'ans1'}, 'q2': {'old': 'ans6', 'new': 'ans2'}} self.assertEqual(expected_result_diff, result_diff)
def create_survey_response(submission, dbm): response = SurveyResponse( dbm, TransportInfo(submission.channel, submission.source, submission.destination), form_code=submission.form_code, form_model_revision=submission.form_model_revision, values=copy(submission.values)) response.create_migrated_response(submission.status, submission.errors, submission.is_void(), submission._doc.submitted_on, submission.test, submission.event_time, submission._doc.data_record_id) return response
def delete(request, project_id): dbm = get_database_manager(request.user) questionnaire = Project.get(dbm, project_id) dashboard_page = settings.HOME_PAGE + "?deleted=true" if questionnaire.is_void(): return HttpResponseRedirect(dashboard_page) survey_response_ids = get_survey_response_ids_from_request(dbm, request, questionnaire) received_times = [] for survey_response_id in survey_response_ids: survey_response = SurveyResponse.get(dbm, survey_response_id) received_times.append(datetime.datetime.strftime(survey_response.submitted_on, "%d/%m/%Y %X")) feeds_dbm = get_feeds_database(request.user) additional_feed_dictionary = get_project_details_dict_for_feed(questionnaire) delete_response = WebPlayerV2(dbm, feeds_dbm).delete_survey_response(survey_response, additional_feed_dictionary, websubmission_logger) mail_feed_errors(delete_response, dbm.database_name) if survey_response.data_record: ReportRouter().delete(get_organization(request).org_id, survey_response.form_code, survey_response.data_record.id) if len(received_times): UserActivityLog().log(request, action=DELETED_DATA_SUBMISSION, project=questionnaire.name, detail=json.dumps({"Date Received": "[%s]" % ", ".join(received_times)})) response = encode_json({'success_message': ugettext("The selected records have been deleted"), 'success': True}) else: response = encode_json({'error_message': ugettext("No records deleted"), 'success': False}) return HttpResponse(response)
def migrate(database): try: log_statement("database: %s" % database) dbm = get_db_manager(SERVER, database=database) revid_map = revsion_map(database, dbm) for old_rev, values in revid_map.iteritems(): survey_response_docs = dbm.database.query( map_survey_response_by_form_model_revision % (values[1], old_rev)) for survey_response_doc in survey_response_docs: survey_response = SurveyResponse.new_from_doc( dbm=dbm, doc=SurveyResponse.__document_class__.wrap( survey_response_doc['value'])) log_statement("Changing revision on:%s from:%s to:%s" % (survey_response.id, survey_response.form_model_revision, values[0])) survey_response.form_model_revision = values[0] survey_response.save() log_statement("Completed Database : %s" % database) mark_as_completed(database) log_file.writelines( "\n=======================================================\n") except Exception: log_statement("Failed Database : %s" % database) traceback.print_exc(file=log_file)
def build_two_error_survey_responses(self): doc_id3 = self.manager._save_document( SurveyResponseDocument(channel="transport", destination=12345, form_model_id=FORM_MODEL_ID, values={ 'Q3': 'ans12', 'Q4': 'ans22' }, status=False, error_message="")) doc_id4 = self.manager._save_document( SurveyResponseDocument(channel="transport", destination=12345, form_model_id="def", values={ 'defQ1': 'defans12', 'defQ2': 'defans22' }, status=False, error_message="")) return [ SurveyResponse.get(self.manager, id) for id in [doc_id3, doc_id4] ]
def build_two_successful_survey_responses(self): doc_id1 = self.manager._save_document( SurveyResponseDocument(channel="transport", destination=12345, form_model_id=FORM_MODEL_ID, values={ 'Q1': 'ans1', 'Q2': 'ans2' }, status=True, error_message="")) doc_id2 = self.manager._save_document( SurveyResponseDocument(channel="transport", destination=12345, form_model_id=FORM_MODEL_ID, values={ 'Q1': 'ans12', 'Q2': 'ans22' }, status=True, error_message="")) return [ SurveyResponse.get(self.manager, id) for id in [doc_id1, doc_id2] ]
def get_survey_responses(dbm, form_model_id, from_time, to_time, page_number=0, page_size=None, view_name="surveyresponse"): startkey, endkey = _get_start_and_end_key(form_model_id, from_time, to_time) if page_size is None: rows = dbm.load_all_rows_in_view(view_name, reduce=False, descending=True, startkey=startkey, endkey=endkey) else: rows = dbm.load_all_rows_in_view(view_name, reduce=False, descending=True, startkey=startkey, endkey=endkey, skip=page_number * page_size, limit=page_size) return [ SurveyResponse.new_from_doc(dbm=dbm, doc=SurveyResponse.__document_class__.wrap( row['value'])) for row in rows ]
def process_submission(database_name, rules, row): manager = get_db_manager(database_name) success = False submission = SurveyResponseDocument._wrap_row(row) for rule in rules: success = success or rule.update_submission(submission) if success: survey_response = SurveyResponse.new_from_doc(manager, submission) survey_response.save()
def build(self, owner_id): survey_response_id = self.manager._save_document( SurveyResponseDocument(channel=self.channel, destination=self.destination, values=self.values, status=self.status, error_message=self.error_message, form_model_id=self.form_model_id, owner_uid=owner_id)) return SurveyResponse.get(self.manager, survey_response_id)
def test_should_return_data_sender_TESTER_when_send_from_TEST_REPORTER_MOBILE_NUMBER( self): survey_response = SurveyResponse( TestDataSenderHelper.manager, TransportInfo("sms", TEST_REPORTER_MOBILE_NUMBER, "destination"), owner_uid=self.test_ds_id) data_sender = get_data_sender(TestDataSenderHelper.manager, survey_response) self.assertEqual(('TEST', 'test', data_sender[2]), data_sender)
def test_should_return_N_A_when_the_data_sender_was_deleted_and_send_from_smart_phone( self): survey_response = SurveyResponse(TestDataSenderHelper.manager, TransportInfo("smartPhone", "*****@*****.**", "destination"), owner_uid=self.deleted_ds_id) data_sender = get_data_sender(TestDataSenderHelper.manager, survey_response) self.assertEqual(("M K Gandhi", u"del1"), data_sender[:2])
def test_should_return_data_sender_information_send_from_web(self): beany_tester_id = get_by_short_code_include_voided( TestDataSenderHelper.manager, "rep1", REPORTER_ENTITY_TYPE).id survey_response = SurveyResponse(TestDataSenderHelper.manager, TransportInfo( "web", "*****@*****.**", "destination"), owner_uid=beany_tester_id) data_sender = get_data_sender(TestDataSenderHelper.manager, survey_response) self.assertEqual(("Beany", "rep1", data_sender[2]), data_sender)
def migrate_db(self): self.logger.info( '\n ============================================= Start =============================================\n') self.logger.info('Start db : %s' % self.dbm.database_name) rows = self.dbm.database.iterview(UNDELETED_SURVEY_RESPONSE, BATCH_SIZE, reduce=False) for row in rows: survey_response_doc = SurveyResponse.__document_class__.wrap(row['value']) survey_response = SurveyResponse.new_from_doc(dbm=self.dbm, doc=survey_response_doc) self._create_feed_doc(survey_response) self.logger.info( '\n ========================================= End ==================================================\n')
def test_update_enriched_survey_response_with_new_survey_response_values(self): field = TextField('name', 'q1', 'A Question') type(self.form_model).fields = PropertyMock(return_value=[field]) type(self.form_model).entity_type = PropertyMock(return_value=['reporter']) self.form_model.id= 'form_model_id' survey_response = SurveyResponse(Mock()) survey_response._doc = SurveyResponseDocument(status=False, values={'q1': 'answer1'}, form_model_id='form_model_id') builder = EnrichedSurveyResponseBuilder(self.dbm, survey_response, self.form_model, {}) def patch_data_sender(): return {} builder._data_sender = patch_data_sender doc = builder.feed_document() self.assertEquals(doc.values, {'q1': 'answer1'}) edited_survey_response_doc = SurveyResponseDocument(status=True, values={'q1': 'answer2'}, form_model_id='form_model_id') edited_survey_response = SurveyResponse(Mock()) edited_survey_response._doc = edited_survey_response_doc new_builder = EnrichedSurveyResponseBuilder(self.dbm, edited_survey_response, self.form_model, {}) new_builder._data_sender = patch_data_sender feeds_dbm = Mock(spec=DatabaseManager) with patch('mangrove.feeds.enriched_survey_response.get_feed_document_by_id') as get_document: get_document.return_value = doc edited_doc = new_builder.update_event_document(feeds_dbm) self.assertEquals(edited_doc.values, {'q1': {'answer': 'answer2', 'label': 'A Question', 'type': 'text'}})
def _get_survey_responses_with_no_eid(dbm, logger): rows = dbm.database.view("surveyresponse/surveyresponse", reduce=False) inconsistent_survey_response_list = [] for row in rows: try: survey_response = SurveyResponse.get(dbm, row.id) form_model = get_form_model_by_code(dbm, survey_response.form_code) if form_model.is_entity_type_reporter( ) and "eid" not in survey_response.values.keys(): inconsistent_survey_response_list.append(survey_response) except Exception as e: logger.exception(e) return inconsistent_survey_response_list
def test_should_return_none_if_survey_response_questionnaire_is_different_from_form_model( self): survey_response_doc = SurveyResponseDocument(values={ 'q5': '23', 'q6': 'sometext', 'q7': 'ab' }) survey_response = SurveyResponse(Mock()) survey_response._doc = survey_response_doc int_field = IntegerField(name='question one', code='q1', label='question one', ddtype=Mock(spec=DataDictType)) text_field = TextField(name='question two', code='q2', label='question two', ddtype=Mock(spec=DataDictType)) choice_field = SelectField(name='question three', code='Q3', label='question three', options=[("one", "a"), ("two", "b"), ("three", "c"), ("four", "d")], single_select_flag=False, ddtype=Mock(spec=DataDictType)) questionnaire_form_model = Mock(spec=FormModel) questionnaire_form_model.form_code = 'test_form_code' questionnaire_form_model.fields = [int_field, text_field, choice_field] result_dict = construct_request_dict(survey_response, questionnaire_form_model) expected_dict = { 'q1': None, 'q2': None, 'Q3': None, 'form_code': 'test_form_code' } self.assertEqual(expected_dict, result_dict)
def migrate_survey_response_with_form_code_as_delete(db_name): logger = logging.getLogger(db_name) logger.info('Starting Migration') manager = get_db_manager(db_name) for row in manager.database.query(survey_responses_with_delete_form_code): try: doc = SurveyResponseDocument.wrap(row['value']) survey_response = SurveyResponse.new_from_doc(manager, doc) logger.info("survey response id: %s" % survey_response.id) survey_response.delete() logger.info("Deleted survey response id: %s" % survey_response.id) except Exception as e: logger.exception("FAILED to delete:%s " % row['value']['_id']) logger.info('Completed Migration')
def test_deep_copy(self): original = SurveyResponse(Mock(), values={'q1': 'ans5', 'q2': 'ans6', 'q3': 'ans3'}, transport_info=TransportInfo('web', 'web', 'web')) duplicate = original.copy() original.values['q1'] = 2 original.values['q2'] = 2 original.values['q3'] = 2 self.assertNotEqual(original.values['q1'], duplicate.values['q1']) self.assertNotEqual(original.values['q2'], duplicate.values['q2']) self.assertNotEqual(original.values['q3'], duplicate.values['q3'])
def _process_survey_response(survey_response_doc, dbm, logger): try: survey_response = SurveyResponse.new_from_doc( dbm=dbm, doc=SurveyResponse.__document_class__.wrap( survey_response_doc['value'])) if 'form_code' not in survey_response._doc: logger.error("form_code not present in survey response:%s" % survey_response.uuid) return None form_code = survey_response._doc['form_code'] form_models = _get_form_models(dbm, survey_response._doc['form_code']) if not form_models: logger.error( "No Questionnaire found for survey response:%s with form_code: %s" % survey_response.uuid, survey_response._doc['form_code']) return None elif len(form_models) > 1: form_models.sort(key=lambda form_model: form_model._doc.created, reverse=True) matching_form_model = _get_matching_form_model( survey_response.created, form_models) if matching_form_model: del survey_response._doc['form_code'] survey_response.form_model_id = matching_form_model.id survey_response.save(process_post_update=False) return form_code else: logger.error( "No Questionnaire found with matching date for survey response: %s and form_code:%s" % survey_response.uuid, survey_response._doc['form_code']) return None elif len(form_models) == 1: del survey_response._doc['form_code'] survey_response.form_model_id = form_models[0].id survey_response.save(process_post_update=False) return form_code except Exception as e: logger.exception("Exception for survey response:%s" % survey_response.uuid) return None
def get_survey_responses_for_activity_period(dbm, form_model_id, from_time, to_time): from_time_in_epoch = convert_date_time_to_epoch( from_time) if from_time is not None else None to_time_in_epoch = convert_date_time_to_epoch( to_time) if to_time is not None else None startkey, endkey = _get_start_and_end_key(form_model_id, from_time_in_epoch, to_time_in_epoch) rows = dbm.load_all_rows_in_view('survey_response_for_activity_period', descending=True, startkey=startkey, endkey=endkey) return [ SurveyResponse.new_from_doc(dbm=dbm, doc=SurveyResponse.__document_class__.wrap( row['value'])) for row in rows ]
def migrate_db(database): mark_as_completed(database) log_statement('\nStart migration on database : %s \n' % database) try: manager = get_db_manager(server=SERVER, database=database) invalid_survey_response_docs = manager.database.query(map_invalid_survey_responses) count = 0 log_statement('Total documents to be voided : %s' % len(invalid_survey_response_docs)) for survey_response_doc in invalid_survey_response_docs: survey_response = SurveyResponse.new_from_doc(dbm=manager, doc=SurveyResponse.__document_class__.wrap(survey_response_doc['value'])) survey_response.void() count += 1 log_statement("Voided survey_response %s" % survey_response.uuid) log_statement("Total number of documents voided are %s" % count) log_statement('\nCompleted database : %s\n' % database) except Exception as e: log_statement('error:%s\n' % database) traceback.print_exc(file=log_file)
def migrate_survey_response_to_add_owner(db_name): logger = logging.getLogger(db_name) try: logger.info('Starting Migration') mark_as_completed(db_name) dbm = get_db_manager(db_name) phone_to_rep_id_map, rep_id_to_uid_map = create_datasender_map(dbm) org_id = OrganizationSetting.objects.get(document_store=dbm.database_name).organization_id source_to_rep_id_map = add_email_info_to_datasender_map(phone_to_rep_id_map, org_id) rows = dbm.database.view("surveyresponse/surveyresponse", reduce=False, include_docs=True) for row in rows: doc_id = row['value']['_id'] try: original_source = row['value']['source'] except KeyError as e: logger.info("Already migrated %s" % row['value']['_id']) #ignore, document already migrated continue doc = SurveyResponseDocument.wrap(row['value']) survey_response = SurveyResponse.new_from_doc(dbm, doc) data_sender_id = source_to_rep_id_map.get(original_source) survey_response.created_by = data_sender_id survey_response.modified_by = data_sender_id owner_short_code = override_owner_with_on_behalf_user(rep_id_to_uid_map, data_sender_id, survey_response) owner_uid = rep_id_to_uid_map.get(owner_short_code) if owner_uid: remove_attr_source_from_survey_response(survey_response) else: logger.warn("Unable to set owner_uid for source :" + original_source + " doc: " + doc_id) survey_response.owner_uid = owner_uid survey_response.save() logger.info("Migrated %s" % survey_response.id) except Exception as e: logger.exception("Failed DB: %s with message %s" % (db_name, e.message)) logger.info('Completed Migration')