def test_generate_elastic_index_for_a_unique_id_field_within_group(self): search_dict = {} unique_id_field = UniqueIdField('clinic', 'my_unique_id', 'my_unique_id', 'My Unique ID') group_field = FieldSet('group_name', 'group_name', 'My Label', field_set=[unique_id_field]) self.form_model.fields = [group_field] value = {'group_name': [{'my_unique_id': 'cli001'}]} submission = SurveyResponseDocument(values=value, status='success') with patch('datawinners.search.submission_index.lookup_entity' ) as lookup_entity: lookup_entity.return_value = {'q2': 'my_clinic'} _update_with_form_model_fields(Mock(spec=DatabaseManager), submission, search_dict, self.form_model) self.assertEqual( search_dict, { '1212_group_name-my_unique_id_details': { 'q2': 'my_clinic' }, '1212_group_name-my_unique_id': 'my_clinic', '1212_group_name-my_unique_id_unique_code': 'cli001', 'is_anonymous': False, 'media': {}, 'void': False })
def test_should_get_comma_separated_list_if_field_changed_from_choice_to_unique_id( self): search_dict = {} options = [{ 'text': 'option1', 'val': 'a' }, { 'text': 'option2', 'val': 'b' }] self.form_model.fields = [self.field1] original_field = SelectField('selectField', 'q1', 'q1', options) self.form_model.get_field_by_code_and_rev.return_value = original_field values = {'q1': 'ab', 'q2': 'wrong number', 'q3': 'wrong text'} submission_doc = SurveyResponseDocument(values=values, status="error") with patch('datawinners.search.submission_index.lookup_entity_name' ) as lookup_entity_name: lookup_entity_name.return_value = 'N/A' _update_with_form_model_fields(Mock(spec=DatabaseManager), submission_doc, search_dict, self.form_model) self.assertEquals( { '1212_q1': 'N/A', '1212_q1_unique_code': 'option1,option2', 'void': False }, search_dict)
def test_should_update_search_dict_with_select_field_in_group_within_repeat( self): search_dict = {} self.form_model.fields = [ FieldSet(name="repeat", code="repeat-code", label="repeat-label", fieldset_type="repeat", field_set=[self.group_question]) ] values = { 'repeat-code': [{ 'group-code': [{ 'q4': 'a b', 'single-select': 'a' }] }] } submission_doc = SurveyResponseDocument(values=values, status="success") _update_with_form_model_fields(Mock(spec=DatabaseManager), submission_doc, search_dict, self.form_model) self.assertEquals( { '1212_repeat-code': '[{"group-code": [{"single-select": "one", "q4": ["one", "two"]}]}]', 'is_anonymous': False, 'void': False }, search_dict)
def create_index(dbm, form_model, logger): form_code = form_model.form_code start_key = [form_code] end_key = [form_code, {}] rows = dbm.database.iterview("surveyresponse/surveyresponse", 1000, reduce=False, include_docs=False, startkey=start_key, endkey=end_key) es = get_elasticsearch_handle(timeout=600) survey_response_docs = [] for row in rows: survey_response = SurveyResponseDocument._wrap_row(row) search_dict = _meta_fields(survey_response, dbm) _update_with_form_model_fields(dbm, survey_response, search_dict, form_model) search_dict.update({'id': survey_response.id}) survey_response_docs.append(search_dict) if survey_response_docs: es.bulk_index(dbm.database_name, form_model.id, survey_response_docs) logger.info('Created index for survey response docs ' + str([doc.get('id') for doc in survey_response_docs]))
def test_should_not_update_search_dict_with_uid_field(self): search_dict = {} self.form_model.fields = [self.field1, self.field2, self.field3] values = {'q2': 'wrong number', 'q3': 'wrong text'} submission_doc = SurveyResponseDocument(values=values, status="error") _update_with_form_model_fields(Mock(spec=DatabaseManager), submission_doc, search_dict, self.form_model) self.assertEquals( { '1212_q2': 'wrong number', '1212_q3': 'wrong text', 'void': False }, search_dict)
def test_generate_elastic_index_for_a_unique_id_field_within_group_in_repeat( self, lookup_entity_mock): unique_id_field = UniqueIdField('clinic', 'my_unique_id', 'my_unique_id', 'My Unique ID') group_field = FieldSet('group_name', 'group_name', 'My Label', field_set=[unique_id_field], fieldset_type='group') repeat_field = FieldSet('repeat_name', 'repeat_name', 'My Label', field_set=[group_field], fieldset_type='repeat') self.form_model.fields = [repeat_field] value = { 'repeat_name': [{ 'group_name': [{ 'my_unique_id': 'cli001' }] }, { 'group_name': [{ 'my_unique_id': 'cli002' }] }] } submission = SurveyResponseDocument(values=value, status='success') lookup_entity_mock.return_value = {'q2': 'my_clinic'} search_dict = {} with patch( 'datawinners.search.submission_index.get_by_short_code_include_voided' ) as get_by_short_code_include_voided_mock: get_by_short_code_include_voided_mock.side_effect = get_by_short_code_include_voided_mock_func _update_with_form_model_fields(Mock(spec=DatabaseManager), submission, search_dict, self.form_model) self.assertDictEqual( search_dict, { '1212_repeat_name': '[{"group_name": [{"my_unique_id": "my_clinic", "my_unique_id_unique_code": "cli001"}]}, ' '{"group_name": [{"my_unique_id": "my_clinic", "my_unique_id_unique_code": "cli002"}]}]', 'is_anonymous': False, 'media': {}, 'void': False, })
def test_should_update_search_dict_with_none_for_missing_entity_answer_in_submission( self): search_dict = {} values = {'q2': 'wrong number', 'q3': 'wrong text'} submission_doc = SurveyResponseDocument(values=values, status="error") _update_with_form_model_fields(Mock(spec=DatabaseManager), submission_doc, search_dict, self.form_model) self.assertEquals( { '1212_eid': 'N/A', "entity_short_code": 'N/A', '1212_q2': 'wrong number', '1212_q3': 'wrong text', 'void': False }, search_dict)
def test_should_update_submission_index_date_field_with_current_format( self): dd_type = Mock(spec=DataDictType) fields = [ TextField(name="entity_question", code="EID", label="What is associated entity", entity_question_flag=True, ddtype=dd_type), DateField("date", "date", "Date", "dd.mm.yyyy", dd_type) ] form_model = FormModel(dbm=Mock(spec=DatabaseManager), form_code="001", type="survey", name="form", entity_type=["clinic"], fields=fields) form_model._doc.entity_type = ["clinic"] values = {'eid': 'cid005', 'date': '12.21.2012'} submission_doc = SurveyResponseDocument(values=values, status="success", form_model_revision="rev1") search_dict = {} form_model._doc = Mock(spec=FormModelDocument) form_model._doc.rev = "rev2" form_model._snapshots = { "rev1": [DateField("date", "date", "Date", "mm.dd.yyyy", dd_type)] } with patch('datawinners.search.submission_index.lookup_entity_name' ) as lookup_entity_name: lookup_entity_name.return_value = 'Test' search_dict = _update_with_form_model_fields( Mock(spec=DatabaseManager), submission_doc, search_dict, form_model) self.assertEquals("21.12.2012", search_dict.get("date"))
def test_generate_elastic_index_for_a_unique_id_field_within_group_in_group( self): unique_id_field = UniqueIdField('clinic', 'my_unique_id', 'my_unique_id', 'My Unique ID') group1_field = FieldSet('group1_name', 'group1_name', 'My Label', field_set=[unique_id_field], fieldset_type='group') group2_field = FieldSet('group2_name', 'group2_name', 'My Label', field_set=[group1_field], fieldset_type='group') self.form_model.fields = [group2_field] value = { 'group2_name': [{ 'group1_name': [{ 'my_unique_id': 'cli001' }] }] } submission = SurveyResponseDocument(values=value, status='success') search_dict = {} with patch('datawinners.search.submission_index.lookup_entity' ) as lookup_entity: with patch( 'datawinners.search.submission_index.get_by_short_code_include_voided' ) as get_by_short_code_include_voided_mock: lookup_entity.return_value = {'q2': 'test'} get_by_short_code_include_voided_mock.side_effect = get_by_short_code_include_voided_mock_func _update_with_form_model_fields(Mock(spec=DatabaseManager), submission, search_dict, self.form_model) self.assertDictEqual( search_dict, { '1212_group1_name-my_unique_id_unique_code': 'cli001', 'is_anonymous': False, 'media': {}, 'void': False, '1212_group1_name-my_unique_id_details': { 'q2': 'test' }, '1212_group1_name-my_unique_id': 'test' })
def test_should_update_search_dict_with_select_field_in_field_set(self): search_dict = {} self.form_model.fields = [self.repeat_question] values = {'repeat-code': [{'q4': 'a b', 'single-select': 'a'}]} submission_doc = SurveyResponseDocument(values=values, status="success") _update_with_form_model_fields(Mock(spec=DatabaseManager), submission_doc, search_dict, self.form_model) self.assertEquals( { '1212_repeat-code': '[{"single-select": "one", "q4": ["one", "two"]}]', 'is_anonymous': False, 'media': {}, 'void': False }, search_dict)
def test_should_update_search_dict_with_select_field_in_group(self): search_dict = {} self.form_model.fields = [self.group_question] values = {'group-code': [{'q4': 'a b', 'single-select': 'a'}]} submission_doc = SurveyResponseDocument(values=values, status="success") self.form_model.get_field_by_code_and_rev.side_effect = lambda code, revision: \ {"q4": self.field4, "single-select": self.field6}[code] _update_with_form_model_fields(Mock(spec=DatabaseManager), submission_doc, search_dict, self.form_model) self.assertEquals( { 'void': False, '1212_group-code-single-select': 'one', '1212_group-code-q4': ['one', 'two'], 'is_anonymous': False }, search_dict)
def test_should_update_search_dict_with_form_field_questions_for_error_submissions( self): search_dict = {} values = {'eid': 'test_id', 'q2': 'wrong number', 'q3': 'wrong text'} submission_doc = EnrichedSurveyResponseDocument(values=values, status="error") with patch('datawinners.search.submission_index.lookup_entity_name' ) as lookup_entity_name: lookup_entity_name.return_value = 'Test' _update_with_form_model_fields(None, submission_doc, search_dict, self.form_model) self.assertEquals( { '1212_eid': 'Test', "entity_short_code": "test_id", '1212_q2': 'wrong number', '1212_q3': 'wrong text', 'void': False }, search_dict)
def test_should_update_search_dict_with_form_field_questions_for_success_submissions( self): search_dict = {} self.form_model.fields = [self.field4] values = {'q4': "ab"} submission_doc = SurveyResponseDocument(values=values, status="success") self.form_model.get_field_by_code_and_rev.return_value = self.field4 with patch('datawinners.search.submission_index.lookup_entity_name' ) as lookup_entity_name: lookup_entity_name.return_value = 'clinic1' _update_with_form_model_fields(Mock(spec=DatabaseManager), submission_doc, search_dict, self.form_model) self.assertEquals({ '1212_q4': ['one', 'two'], 'void': False }, search_dict)
def test_should_update_search_dict_with_form_field_questions_for_error_submissions( self): search_dict = {} self.form_model.fields = [self.field1, self.field2, self.field3] values = {'q1': 'test_id', 'q2': 'wrong number', 'q3': 'wrong text'} submission_doc = SurveyResponseDocument(values=values, status="error") with patch('datawinners.search.submission_index.lookup_entity_name' ) as lookup_entity_name: lookup_entity_name.return_value = 'test1' _update_with_form_model_fields(None, submission_doc, search_dict, self.form_model) self.assertEquals( { '1212_q1': 'test1', "1212_q1_unique_code": "test_id", '1212_q2': 'wrong number', '1212_q3': 'wrong text', 'is_anonymous': False, 'void': False }, search_dict)
def test_generate_elastic_index_for_a_unique_id_field_in_parent_level( self): search_dict = {} self.form_model.fields = [ UniqueIdField('clinic', 'my_unique_id', 'my_unique_id', 'My Unique ID') ] value = {'my_unique_id': 'cli001'} submission = SurveyResponseDocument(values=value, status='success') with patch('datawinners.search.submission_index.lookup_entity_name' ) as lookup_entity_name: lookup_entity_name.return_value = 'my_clinic' _update_with_form_model_fields(Mock(spec=DatabaseManager), submission, search_dict, self.form_model) self.assertEqual( search_dict, { '1212_my_unique_id': 'my_clinic', '1212_my_unique_id_unique_code': 'cli001', 'is_anonymous': False, 'void': False })
def test_should_update_search_dict_with_form_field_questions_for_success_submissions( self): search_dict = {} mock_select_field = Mock( spec=Field, is_entity_field=False, code="q4", type="select", get_option_value_list=Mock(return_value=["one", "two"])) self.form_model.fields.append(mock_select_field) self.form_model.fields.append( Mock(spec=Field, is_entity_field=False, code="q5", type="text")) values = { 'eid': 'cid005', 'q2': "name", 'q3': "3,3", 'q4': "ab", 'q5': '11.12.2012' } submission_doc = SurveyResponseDocument(values=values, status="success") self.form_model.get_field_by_code_and_rev.return_value = mock_select_field with patch('datawinners.search.submission_index.lookup_entity_name' ) as lookup_entity_name: lookup_entity_name.return_value = 'Test' _update_with_form_model_fields(None, submission_doc, search_dict, self.form_model) self.assertEquals( { '1212_eid': 'Test', "entity_short_code": "cid005", '1212_q2': 'name', '1212_q3': '3,3', '1212_q4': ['one', 'two'], '1212_q5': '11.12.2012', 'void': False }, search_dict)
def create_submission_index(dbm, row): form_model = Project.new_from_doc(dbm, ProjectDocument.wrap(row["value"])) form_code = form_model.form_code start_key = [form_code] end_key = [form_code, {}] rows = dbm.database.iterview("surveyresponse/surveyresponse", 1000, reduce=False, include_docs=False, startkey=start_key, endkey=end_key) es = get_elasticsearch_handle(timeout=600) survey_response_docs = [] for row in rows: survey_response = SurveyResponseDocument._wrap_row(row) search_dict = _meta_fields(survey_response, dbm) _update_with_form_model_fields(dbm, survey_response, search_dict, form_model) search_dict.update({'id': survey_response.id}) survey_response_docs.append(search_dict) if survey_response_docs: es.bulk_index(dbm.database_name, form_model.id, survey_response_docs)
def test_should_update_submission_index_date_field_with_current_format( self): self.form_model.fields = [self.field1, self.field5] values = {'q1': 'cid005', 'q5': '12.2012'} submission_doc = SurveyResponseDocument(values=values, status="success", form_model_revision="rev1") search_dict = {} self.form_model._doc = Mock(spec=FormModelDocument) self.form_model._doc.rev = "rev2" self.form_model.get_field_by_code_and_rev.return_value = DateField( "q5", "date", "Date", "mm.yyyy") with patch('datawinners.search.submission_index.lookup_entity_name' ) as lookup_entity_name: lookup_entity_name.return_value = 'Test' search_dict = _update_with_form_model_fields( Mock(spec=DatabaseManager), submission_doc, search_dict, self.form_model) self.assertEquals("12.01.2012", search_dict.get("1212_q5"))