def test_should_update_submission_index_date_field_with_current_format(
self):
dd_type = Mock(spec=DataDictType)
fields = [
TextField(name="entity_question",
code="EID",
label="What is associated entity",
entity_question_flag=True,
ddtype=dd_type),
DateField("date", "date", "Date", "dd.mm.yyyy", dd_type)
]
form_model = FormModel(dbm=Mock(spec=DatabaseManager),
form_code="001",
type="survey",
name="form",
entity_type=["clinic"],
fields=fields)
form_model._doc.entity_type = ["clinic"]
values = {'eid': 'cid005', 'date': '12.21.2012'}
submission_doc = SurveyResponseDocument(values=values,
status="success",
form_model_revision="rev1")
search_dict = {}
form_model._doc = Mock(spec=FormModelDocument)
form_model._doc.rev = "rev2"
form_model._snapshots = {
"rev1": [DateField("date", "date", "Date", "mm.dd.yyyy", dd_type)]
}
with patch('datawinners.search.submission_index.lookup_entity_name'
) as lookup_entity_name:
lookup_entity_name.return_value = 'Test'
search_dict = _update_with_form_model_fields(
Mock(spec=DatabaseManager), submission_doc, search_dict,
form_model)
self.assertEquals("21.12.2012", search_dict.get("date"))
def test_should_return_none_if_survey_response_questionnaire_is_different_from_form_model(
self):
survey_response_doc = SurveyResponseDocument(values={
'q5': '23',
'q6': 'sometext',
'q7': 'ab'
})
survey_response = SurveyResponse(Mock())
survey_response._doc = survey_response_doc
int_field = IntegerField(name='question one',
code='q1',
label='question one')
text_field = TextField(name='question two',
code='q2',
label='question two')
choice_field = SelectField(name='question three',
code='Q3',
label='question three',
options=[("one", "a"), ("two", "b"),
("three", "c"), ("four", "d")],
single_select_flag=False)
questionnaire_form_model = Mock(spec=FormModel)
questionnaire_form_model.form_code = 'test_form_code'
questionnaire_form_model.fields = [int_field, text_field, choice_field]
result_dict = construct_request_dict(survey_response,
questionnaire_form_model, 'id')
expected_dict = {
'q1': None,
'q2': None,
'Q3': None,
'form_code': 'test_form_code',
'dsid': 'id'
}
self.assertEqual(expected_dict, result_dict)
def test_should_update_search_dict_with_select_field_in_group_within_repeat(
self):
search_dict = {}
self.form_model.fields = [
FieldSet(name="repeat",
code="repeat-code",
label="repeat-label",
fieldset_type="repeat",
field_set=[self.group_question])
]
values = {
'repeat-code': [{
'group-code': [{
'q4': 'a b',
'single-select': 'a'
}]
}]
}
submission_doc = SurveyResponseDocument(values=values,
status="success")
_update_with_form_model_fields(Mock(spec=DatabaseManager),
submission_doc, search_dict,
self.form_model)
self.assertEquals(
{
'1212_repeat-code':
'[{"group-code": [{"single-select": "one", "q4": ["one", "two"]}]}]',
'is_anonymous': False,
'void': False
}, search_dict)
def test_should_get_comma_separated_list_if_field_changed_from_choice_to_unique_id(
self):
search_dict = {}
options = [{
'text': 'option1',
'val': 'a'
}, {
'text': 'option2',
'val': 'b'
}]
self.form_model.fields = [self.field1]
original_field = SelectField('selectField', 'q1', 'q1', options)
self.form_model.get_field_by_code_and_rev.return_value = original_field
values = {'q1': 'ab', 'q2': 'wrong number', 'q3': 'wrong text'}
submission_doc = SurveyResponseDocument(values=values, status="error")
with patch('datawinners.search.submission_index.lookup_entity_name'
) as lookup_entity_name:
lookup_entity_name.return_value = 'N/A'
_update_with_form_model_fields(Mock(spec=DatabaseManager),
submission_doc, search_dict,
self.form_model)
self.assertEquals(
{
'1212_q1': 'N/A',
'1212_q1_unique_code': 'option1,option2',
'void': False
}, search_dict)
def test_generate_elastic_index_for_a_unique_id_field_within_group(self):
search_dict = {}
unique_id_field = UniqueIdField('clinic', 'my_unique_id',
'my_unique_id', 'My Unique ID')
group_field = FieldSet('group_name',
'group_name',
'My Label',
field_set=[unique_id_field])
self.form_model.fields = [group_field]
value = {'group_name': [{'my_unique_id': 'cli001'}]}
submission = SurveyResponseDocument(values=value, status='success')
with patch('datawinners.search.submission_index.lookup_entity'
) as lookup_entity:
lookup_entity.return_value = {'q2': 'my_clinic'}
_update_with_form_model_fields(Mock(spec=DatabaseManager),
submission, search_dict,
self.form_model)
self.assertEqual(
search_dict, {
'1212_group_name-my_unique_id_details': {
'q2': 'my_clinic'
},
'1212_group_name-my_unique_id': 'my_clinic',
'1212_group_name-my_unique_id_unique_code': 'cli001',
'is_anonymous': False,
'media': {},
'void': False
})
def create_index(dbm, form_model, logger):
form_code = form_model.form_code
start_key = [form_code]
end_key = [form_code, {}]
rows = dbm.database.iterview("surveyresponse/surveyresponse",
1000,
reduce=False,
include_docs=False,
startkey=start_key,
endkey=end_key)
es = get_elasticsearch_handle(timeout=600)
survey_response_docs = []
for row in rows:
survey_response = SurveyResponseDocument._wrap_row(row)
search_dict = _meta_fields(survey_response, dbm)
_update_with_form_model_fields(dbm, survey_response, search_dict,
form_model)
search_dict.update({'id': survey_response.id})
survey_response_docs.append(search_dict)
if survey_response_docs:
es.bulk_index(dbm.database_name, form_model.id, survey_response_docs)
logger.info('Created index for survey response docs ' +
str([doc.get('id') for doc in survey_response_docs]))
def copy(self):
survey_copy = SurveyResponse(None)
survey_copy._doc = SurveyResponseDocument(self._doc.channel, self._doc.destination,
deepcopy(self.values), self.id, self.status, self.errors,
self.form_model_id, self.form_model_revision,
self.data_record.id if self.data_record else None,
deepcopy(self.event_time))
return survey_copy
def process_submission(database_name, rules, row):
manager = get_db_manager(database_name)
success = False
submission = SurveyResponseDocument._wrap_row(row)
for rule in rules:
success = success or rule.update_submission(submission)
if success:
survey_response = SurveyResponse.new_from_doc(manager, submission)
survey_response.save()
def load_sms_data():
dbm = load_manager_for_default_test_account()
for i in range(1, 200):
sender = "1234567890"
receiver = 12345
dbm._save_document(SurveyResponseDocument(channel="sms",
destination=receiver, form_code="cli001",
values={"EID": "CID001", "Q1": "prabhu", "Q2": 20},
status=True, error_message=""))
def test_get_submissions_for_form_for_an_activity_period(self):
self.manager._save_document(
SurveyResponseDocument(channel="web",
destination="",
form_model_id="abc",
values={
'Q1': 'ans1',
'Q2': 'ans2'
},
status=False,
error_message="",
data_record_id='2345678',
event_time=datetime.datetime(2011, 9, 1)))
self.manager._save_document(
SurveyResponseDocument(channel="web",
destination="",
form_model_id="abc",
values={
'Q1': 'ans12',
'Q2': 'ans22'
},
status=False,
error_message="",
data_record_id='1234567',
event_time=datetime.datetime(2011, 3, 3)))
self.manager._save_document(
SurveyResponseDocument(channel="web",
destination="",
form_model_id="abc",
values={
'Q1': 'ans12',
'Q2': 'defans22'
},
status=False,
error_message="",
data_record_id='345678',
event_time=datetime.datetime(2011, 3, 10)))
from_time = datetime.datetime(2011, 3, 1)
end_time = datetime.datetime(2011, 3, 30)
survey_responses = get_survey_responses_for_activity_period(
self.manager, "abc", from_time, end_time)
self.assertEquals(2, len(survey_responses))
def build(self, owner_id):
survey_response_id = self.manager._save_document(
SurveyResponseDocument(channel=self.channel,
destination=self.destination,
values=self.values,
status=self.status,
error_message=self.error_message,
form_model_id=self.form_model_id,
owner_uid=owner_id))
return SurveyResponse.get(self.manager, survey_response_id)
def _prepare_survey_responses(self):
doc_id1 = self.manager._save_document(
SurveyResponseDocument(channel="transport",
destination=12345,
form_model_id=FORM_MODEL_ID,
values={
'Q1': 'ans1',
'Q2': 'ans2'
},
status=True,
error_message=""))
doc_id2 = self.manager._save_document(
SurveyResponseDocument(channel="transport",
destination=12345,
form_model_id=FORM_MODEL_ID,
values={
'Q1': 'ans12',
'Q2': 'ans22'
},
status=True,
error_message=""))
doc_id3 = self.manager._save_document(
SurveyResponseDocument(channel="transport",
destination=12345,
form_model_id=FORM_MODEL_ID,
values={
'Q3': 'ans12',
'Q4': 'ans22'
},
status=False,
error_message=""))
doc_id4 = self.manager._save_document(
SurveyResponseDocument(channel="transport",
destination=12345,
form_model_id="def",
values={
'defQ1': 'defans12',
'defQ2': 'defans22'
},
status=False,
error_message=""))
return [doc_id1, doc_id2, doc_id3, doc_id4]
def test_should_update_the_associated_submission_when_question_code_is_updated(self):
update_dict = {"database_name": "database_name", "old_form_code": "old_form_code",
"new_form_code": "new_form_code", "new_revision": "new_revision"}
with patch("datawinners.project.wizard_view.get_db_manager") as get_db_manager:
managerMock = Mock(spec=DatabaseManager)
get_db_manager.return_value = managerMock
managerMock._save_documents.return_value = []
with patch(
"datawinners.project.wizard_view.survey_responses_by_form_code") as survey_responses_by_form_code:
mock_document1 = SurveyResponseDocument()
mock_document2 = SurveyResponseDocument()
survey_responses_mock = [(SurveyResponse.new_from_doc(dbm=None, doc=mock_document1)),
(SurveyResponse.new_from_doc(dbm=None, doc=mock_document2))]
survey_responses_by_form_code.return_value = survey_responses_mock
update_submissions_for_form_code_change(managerMock, 'new_form_code', 'old_form_code')
managerMock._save_documents.assert_called_with([mock_document1, mock_document2])
self.assertEquals(mock_document1.form_code, "new_form_code")
self.assertEquals(mock_document2.form_code, "new_form_code")
def test_convert_survey_response_values_to_dict_format(self):
survey_response_doc = SurveyResponseDocument(
values={
'q1': '23',
'q2': 'sometext',
'q3': 'a',
'geo': '2.34,5.64',
'date': '12.12.2012'
})
survey_response = SurveyResponse(Mock())
survey_response._doc = survey_response_doc
int_field = IntegerField(name='question one',
code='q1',
label='question one',
ddtype=Mock(spec=DataDictType))
text_field = TextField(name='question two',
code='q2',
label='question two',
ddtype=Mock(spec=DataDictType))
single_choice_field = SelectField(name='question three',
code='q3',
label='question three',
options=[("one", "a"), ("two", "b"),
("three", "c"),
("four", "d")],
single_select_flag=True,
ddtype=Mock(spec=DataDictType))
geo_field = GeoCodeField(name='geo',
code='GEO',
label='geo',
ddtype=Mock())
date_field = DateField(name='date',
code='DATE',
label='date',
date_format='dd.mm.yyyy',
ddtype=Mock())
questionnaire_form_model = Mock(spec=FormModel)
questionnaire_form_model.form_code = 'test_form_code'
questionnaire_form_model.fields = [
int_field, text_field, single_choice_field, geo_field, date_field
]
request_dict = construct_request_dict(survey_response,
questionnaire_form_model)
expected_dict = OrderedDict({
'q1': '23',
'q2': 'sometext',
'q3': 'a',
'GEO': '2.34,5.64',
'DATE': '12.12.2012',
'form_code': 'test_form_code'
})
self.assertEqual(expected_dict, request_dict)
def __init__(self, dbm, transport_info=None, form_model_id=None, form_model_revision=None, values=None, owner_uid=None,
admin_id=None, response=None):
DataObject.__init__(self, dbm)
self.response = response
if transport_info is not None:
doc = SurveyResponseDocument(channel=transport_info.transport,
destination=transport_info.destination,
form_model_id=form_model_id,
form_model_revision=form_model_revision,
values=values, status=False,
error_message="", owner_uid=owner_uid, modified_by_id=admin_id)
DataObject._set_document(self, doc)
def test_should_get_static_info_from_submission(self):
with patch("datawinners.project.views.submission_views.get_data_sender"
) as get_data_sender:
survey_response_document = SurveyResponseDocument(
channel='web',
status=False,
error_message="Some Error in submission")
get_data_sender.return_value = ('Psub', 'rep2')
submission_date = datetime(2012, 02, 20, 12, 15, 44)
survey_response_document.submitted_on = submission_date
survey_response_document.created = datetime(
2012, 02, 20, 12, 15, 50)
survey_response = SurveyResponse(Mock())
survey_response._doc = survey_response_document
project = Mock()
project.data_senders = ["rep2"]
organization_mock = Mock()
organization_mock.org_id = "TEST1234"
# with patch("datawinners.project.views.submission_views.get_organization_from_manager") as get_ngo_from_manager_mock:
# get_ngo_from_manager_mock.return_value = organization_mock
static_info = build_static_info_context(
Mock(), survey_response, questionnaire_form_model=project)
expected_values = OrderedDict({
'static_content': {
'Data Sender': ('Psub', 'rep2'),
'Source':
u'Web',
'Submission Date':
submission_date.strftime(
SUBMISSION_DATE_FORMAT_FOR_SUBMISSION)
}
})
expected_values.update({'is_edit': True})
expected_values.update({'status': u'Error'})
self.assertEqual(expected_values, static_info)
def test_should_not_update_search_dict_with_uid_field(self):
search_dict = {}
self.form_model.fields = [self.field1, self.field2, self.field3]
values = {'q2': 'wrong number', 'q3': 'wrong text'}
submission_doc = SurveyResponseDocument(values=values, status="error")
_update_with_form_model_fields(Mock(spec=DatabaseManager),
submission_doc, search_dict,
self.form_model)
self.assertEquals(
{
'1212_q2': 'wrong number',
'1212_q3': 'wrong text',
'void': False
}, search_dict)
def migrate_survey_response_with_form_code_as_delete(db_name):
logger = logging.getLogger(db_name)
logger.info('Starting Migration')
manager = get_db_manager(db_name)
for row in manager.database.query(survey_responses_with_delete_form_code):
try:
doc = SurveyResponseDocument.wrap(row['value'])
survey_response = SurveyResponse.new_from_doc(manager, doc)
logger.info("survey response id: %s" % survey_response.id)
survey_response.delete()
logger.info("Deleted survey response id: %s" % survey_response.id)
except Exception as e:
logger.exception("FAILED to delete:%s " % row['value']['_id'])
logger.info('Completed Migration')
def populate_submission_index(dbm, form_code=None):
start_key = [form_code] if form_code else []
end_key = [form_code, {}] if form_code else [{}, {}]
rows = dbm.database.iterview("surveyresponse/surveyresponse", 1000, reduce=False, include_docs=False, startkey=start_key, endkey=end_key)
logger = logging.getLogger(dbm.database_name)
ignored = 0
for row in rows:
try:
survey_response = SurveyResponseDocument._wrap_row(row)
update_submission_search_index(survey_response, dbm, refresh_index=False)
except FormModelDoesNotExistsException as e:
ignored += 1
logger.warning(e.message) # ignore orphaned submissions On changing form code!
if ignored > 0:
logger.warning("Few submissions are ignored %s" % ignored)
def test_generate_elastic_index_for_a_unique_id_field_within_group_in_repeat(
self, lookup_entity_mock):
unique_id_field = UniqueIdField('clinic', 'my_unique_id',
'my_unique_id', 'My Unique ID')
group_field = FieldSet('group_name',
'group_name',
'My Label',
field_set=[unique_id_field],
fieldset_type='group')
repeat_field = FieldSet('repeat_name',
'repeat_name',
'My Label',
field_set=[group_field],
fieldset_type='repeat')
self.form_model.fields = [repeat_field]
value = {
'repeat_name': [{
'group_name': [{
'my_unique_id': 'cli001'
}]
}, {
'group_name': [{
'my_unique_id': 'cli002'
}]
}]
}
submission = SurveyResponseDocument(values=value, status='success')
lookup_entity_mock.return_value = {'q2': 'my_clinic'}
search_dict = {}
with patch(
'datawinners.search.submission_index.get_by_short_code_include_voided'
) as get_by_short_code_include_voided_mock:
get_by_short_code_include_voided_mock.side_effect = get_by_short_code_include_voided_mock_func
_update_with_form_model_fields(Mock(spec=DatabaseManager),
submission, search_dict,
self.form_model)
self.assertDictEqual(
search_dict, {
'1212_repeat_name':
'[{"group_name": [{"my_unique_id": "my_clinic", "my_unique_id_unique_code": "cli001"}]}, '
'{"group_name": [{"my_unique_id": "my_clinic", "my_unique_id_unique_code": "cli002"}]}]',
'is_anonymous':
False,
'media': {},
'void':
False,
})
def test_should_update_search_dict_with_none_for_missing_entity_answer_in_submission(
self):
search_dict = {}
values = {'q2': 'wrong number', 'q3': 'wrong text'}
submission_doc = SurveyResponseDocument(values=values, status="error")
_update_with_form_model_fields(Mock(spec=DatabaseManager),
submission_doc, search_dict,
self.form_model)
self.assertEquals(
{
'1212_eid': 'N/A',
"entity_short_code": 'N/A',
'1212_q2': 'wrong number',
'1212_q3': 'wrong text',
'void': False
}, search_dict)
def test_should_update_search_dict_with_select_field_in_field_set(self):
search_dict = {}
self.form_model.fields = [self.repeat_question]
values = {'repeat-code': [{'q4': 'a b', 'single-select': 'a'}]}
submission_doc = SurveyResponseDocument(values=values,
status="success")
_update_with_form_model_fields(Mock(spec=DatabaseManager),
submission_doc, search_dict,
self.form_model)
self.assertEquals(
{
'1212_repeat-code':
'[{"single-select": "one", "q4": ["one", "two"]}]',
'is_anonymous': False,
'media': {},
'void': False
}, search_dict)
def test_generate_elastic_index_for_a_unique_id_field_within_group_in_group(
self):
unique_id_field = UniqueIdField('clinic', 'my_unique_id',
'my_unique_id', 'My Unique ID')
group1_field = FieldSet('group1_name',
'group1_name',
'My Label',
field_set=[unique_id_field],
fieldset_type='group')
group2_field = FieldSet('group2_name',
'group2_name',
'My Label',
field_set=[group1_field],
fieldset_type='group')
self.form_model.fields = [group2_field]
value = {
'group2_name': [{
'group1_name': [{
'my_unique_id': 'cli001'
}]
}]
}
submission = SurveyResponseDocument(values=value, status='success')
search_dict = {}
with patch('datawinners.search.submission_index.lookup_entity'
) as lookup_entity:
with patch(
'datawinners.search.submission_index.get_by_short_code_include_voided'
) as get_by_short_code_include_voided_mock:
lookup_entity.return_value = {'q2': 'test'}
get_by_short_code_include_voided_mock.side_effect = get_by_short_code_include_voided_mock_func
_update_with_form_model_fields(Mock(spec=DatabaseManager),
submission, search_dict,
self.form_model)
self.assertDictEqual(
search_dict, {
'1212_group1_name-my_unique_id_unique_code': 'cli001',
'is_anonymous': False,
'media': {},
'void': False,
'1212_group1_name-my_unique_id_details': {
'q2': 'test'
},
'1212_group1_name-my_unique_id': 'test'
})
def test_should_update_search_dict_with_select_field_in_group(self):
search_dict = {}
self.form_model.fields = [self.group_question]
values = {'group-code': [{'q4': 'a b', 'single-select': 'a'}]}
submission_doc = SurveyResponseDocument(values=values,
status="success")
self.form_model.get_field_by_code_and_rev.side_effect = lambda code, revision: \
{"q4": self.field4, "single-select": self.field6}[code]
_update_with_form_model_fields(Mock(spec=DatabaseManager),
submission_doc, search_dict,
self.form_model)
self.assertEquals(
{
'void': False,
'1212_group-code-single-select': 'one',
'1212_group-code-q4': ['one', 'two'],
'is_anonymous': False
}, search_dict)
def test_should_replace_answer_option_values_with_options_text_when_answer_type_is_changed_from_single_select_choice_field(
self):
survey_response_doc = SurveyResponseDocument(values={'q1': 'a', })
survey_response = SurveyResponse(Mock())
survey_response._doc = survey_response_doc
choice_field = SelectField(name='question one', code='q1', label='question one',
options=[("one", "a"), ("two", "b"), ("three", "c"), ("four", "d")],
single_select_flag=True)
text_field = TextField(name='question one', code='q1', label='question one')
questionnaire_form_model = Mock(spec=FormModel)
questionnaire_form_model.form_code = 'test_form_code'
questionnaire_form_model.fields = [text_field]
questionnaire_form_model.get_field_by_code_and_rev.return_value = choice_field
request_dict = construct_request_dict(survey_response, questionnaire_form_model, 'id')
expected_dict = {'q1': 'one', 'form_code': 'test_form_code', 'dsid': 'id'}
self.assertEqual(request_dict, expected_dict)
def test_should_update_submission_index_date_field_with_current_format(
self):
self.form_model.fields = [self.field1, self.field5]
values = {'q1': 'cid005', 'q5': '12.2012'}
submission_doc = SurveyResponseDocument(values=values,
status="success",
form_model_revision="rev1")
search_dict = {}
self.form_model._doc = Mock(spec=FormModelDocument)
self.form_model._doc.rev = "rev2"
self.form_model.get_field_by_code_and_rev.return_value = DateField(
"q5", "date", "Date", "mm.yyyy")
with patch('datawinners.search.submission_index.lookup_entity_name'
) as lookup_entity_name:
lookup_entity_name.return_value = 'Test'
search_dict = _update_with_form_model_fields(
Mock(spec=DatabaseManager), submission_doc, search_dict,
self.form_model)
self.assertEquals("12.01.2012", search_dict.get("1212_q5"))
def populate_submission_index(dbm, form_model_id=None):
logger = logging.getLogger()
if form_model_id is None:
questionnaires = dbm.load_all_rows_in_view("surveyresponse_by_questionnaire_id", reduce=True, group=True)
for q in questionnaires:
logger.info('Processing questionnaire id {q}'.format(q=q.key))
populate_submission_index(dbm, q.key)
else:
start = time.time()
rows = get_survey_responses_by_form_model_id(dbm, form_model_id)
form_model = FormModel.get(dbm, form_model_id)
logger = logging.getLogger(form_model.name)
ignored = 0
counter = 0
error_count = 0
actions = []
es = get_elasticsearch_handle()
for row in rows:
try:
survey_response = SurveyResponseDocument._wrap_row(row)
submission_action = update_submission_search_index(survey_response, dbm, refresh_index=False,
form_model=form_model, bulk=True)
actions.append(submission_action)
if len(actions) == settings.ES_INDEX_RECREATION_BATCH:
es.bulk(actions, index=dbm.database_name, doc_type=form_model.id)
actions = []
counter += 1
logger.info('No of submissions processed {counter}'.format(counter=counter))
except FormModelDoesNotExistsException as e:
ignored += 1
logger.warning(e.message) # ignore orphaned submissions On changing form code!
except Exception as ex:
logger.exception('Exception occurred')
error_count += 1
if len(actions) > 0:
es.bulk(actions, index=dbm.database_name, doc_type=form_model.id)
logger.warning("No of submissions ignored: {ignored}".format(ignored=ignored))
logger.warning("No of submissions had errors:{errors}".format(errors=error_count))
logger.info('Time taken (seconds) for indexing {counter} submissions of questionnaire {q} : {timetaken}'
.format(counter=counter,q=form_model_id,timetaken=(time.time()-start)))
def migrate_survey_response_to_add_owner(db_name):
logger = logging.getLogger(db_name)
try:
logger.info('Starting Migration')
mark_as_completed(db_name)
dbm = get_db_manager(db_name)
phone_to_rep_id_map, rep_id_to_uid_map = create_datasender_map(dbm)
org_id = OrganizationSetting.objects.get(document_store=dbm.database_name).organization_id
source_to_rep_id_map = add_email_info_to_datasender_map(phone_to_rep_id_map, org_id)
rows = dbm.database.view("surveyresponse/surveyresponse", reduce=False, include_docs=True)
for row in rows:
doc_id = row['value']['_id']
try:
original_source = row['value']['source']
except KeyError as e:
logger.info("Already migrated %s" % row['value']['_id']) #ignore, document already migrated
continue
doc = SurveyResponseDocument.wrap(row['value'])
survey_response = SurveyResponse.new_from_doc(dbm, doc)
data_sender_id = source_to_rep_id_map.get(original_source)
survey_response.created_by = data_sender_id
survey_response.modified_by = data_sender_id
owner_short_code = override_owner_with_on_behalf_user(rep_id_to_uid_map, data_sender_id, survey_response)
owner_uid = rep_id_to_uid_map.get(owner_short_code)
if owner_uid:
remove_attr_source_from_survey_response(survey_response)
else:
logger.warn("Unable to set owner_uid for source :" + original_source + " doc: " + doc_id)
survey_response.owner_uid = owner_uid
survey_response.save()
logger.info("Migrated %s" % survey_response.id)
except Exception as e:
logger.exception("Failed DB: %s with message %s" % (db_name, e.message))
logger.info('Completed Migration')
def test_should_update_search_dict_with_form_field_questions_for_error_submissions(
self):
search_dict = {}
self.form_model.fields = [self.field1, self.field2, self.field3]
values = {'q1': 'test_id', 'q2': 'wrong number', 'q3': 'wrong text'}
submission_doc = SurveyResponseDocument(values=values, status="error")
with patch('datawinners.search.submission_index.lookup_entity_name'
) as lookup_entity_name:
lookup_entity_name.return_value = 'test1'
_update_with_form_model_fields(None, submission_doc, search_dict,
self.form_model)
self.assertEquals(
{
'1212_q1': 'test1',
"1212_q1_unique_code": "test_id",
'1212_q2': 'wrong number',
'1212_q3': 'wrong text',
'is_anonymous': False,
'void': False
}, search_dict)
def test_should_update_search_dict_with_form_field_questions_for_success_submissions(
self):
search_dict = {}
self.form_model.fields = [self.field4]
values = {'q4': "ab"}
submission_doc = SurveyResponseDocument(values=values,
status="success")
self.form_model.get_field_by_code_and_rev.return_value = self.field4
with patch('datawinners.search.submission_index.lookup_entity_name'
) as lookup_entity_name:
lookup_entity_name.return_value = 'clinic1'
_update_with_form_model_fields(Mock(spec=DatabaseManager),
submission_doc, search_dict,
self.form_model)
self.assertEquals({
'1212_q4': ['one', 'two'],
'void': False
}, search_dict)