def is_mapping_out_of_sync(form_model_doc, dbm):
form_model = FormModel.new_from_doc(dbm, form_model_doc)
if form_model.form_code == 'delete':
return
submission_search_store = SubmissionSearchStore(dbm,
form_model,
old_form_model=None)
current_mapping = submission_search_store.get_mappings()
old_mapping = submission_search_store.get_old_mappings()
return submission_search_store.is_mapping_out_of_sync()
def async_reindex(db_name, form_model_id):
response = dict()
response['db_name'] = db_name
response['questionnaire_id'] = form_model_id
response['start_time'] = time.time()
dbm = get_db_manager(db_name)
form_model = FormModel.get(dbm, form_model_id)
submission_search_store = SubmissionSearchStore(dbm, form_model, old_form_model=None)
submission_search_store.recreate_elastic_store()
_populate_submission_index(db_name, form_model_id)
response['end_time'] = time.time()
return response
def check_mapping_out_of_sync(form_model, dbm):
logger = logging.getLogger('datawinners.tasks')
if form_model.form_code == 'delete':
return
try:
submission_search_store = SubmissionSearchStore(dbm,
form_model,
old_form_model=None)
current_mapping = submission_search_store.get_mappings()
old_mapping = submission_search_store.get_old_mappings()
return submission_search_store.is_mapping_out_of_sync()
except Exception as e:
logger.exception('Skipping form model ' + form_model.id)
return False #Retrigger after fixing any environment issue
def test_should_throw_exception_when_unique_id_type_field_code_changes(
self):
new_fields = [
DateField(name='q3',
code='q3',
label='Reporting date',
date_format='dd.mm.yyyy'),
UniqueIdField(unique_id_type='clinic',
name="Q1",
code="EID new",
label="What is the clinic id?")
]
latest_form_model = FormModel(self.dbm,
"abc",
"abc",
form_code="cli001",
fields=new_fields)
es_mock = MagicMock()
with patch(
"datawinners.search.submission_index.get_elasticsearch_handle"
) as get_elasticsearch_handle_mock:
with self.assertRaises(FieldTypeChangeException) as e:
get_elasticsearch_handle_mock.return_value = es_mock
SubmissionSearchStore(
self.dbm, latest_form_model,
self.form_model)._verify_unique_id_change()
def test_should_not_throw_exception_when_unique_id_field_reordered(self):
new_fields = [
UniqueIdField(unique_id_type='clinic',
name="Q1",
code="EID",
label="What is the clinic id?"),
DateField(name='q3',
code='q3',
label='Reporting date',
date_format='dd.mm.yyyy')
]
es_mock = MagicMock()
latest_form_model = FormModel(self.dbm,
"abc",
"abc",
form_code="cli001",
fields=new_fields)
with patch(
"datawinners.search.submission_index.get_elasticsearch_handle"
) as get_elasticsearch_handle_mock:
get_elasticsearch_handle_mock.return_value = es_mock
SubmissionSearchStore(self.dbm, latest_form_model,
self.form_model)._verify_unique_id_change()
self.assertFalse(self.es.put_mapping.called)
def test_should_add_es_mapping_when_no_existing_questions_mapping(self):
no_old_mapping = []
self.es.get_mapping.return_value = no_old_mapping
SubmissionSearchStore(self.dbm, self.es, self.form_model,
self.form_model).update_store()
self.assertTrue(self.es.put_mapping.called)
def test_should_add_es_mapping_when_no_existing_questions_mapping(self):
dbm = MagicMock(spec=DatabaseManager)
dbm.database_name = 'somedb'
es = Mock()
no_old_mapping = []
es.get_mapping.return_value = no_old_mapping
SubmissionSearchStore(dbm, es, self.form_model).update_store()
self.assertTrue(es.put_mapping.called)
def test_should_add_es_mapping_when_no_existing_questions_mapping(self):
no_old_mapping = []
es_mock = MagicMock()
with patch(
"datawinners.search.submission_index.get_elasticsearch_handle"
) as get_elasticsearch_handle_mock:
get_elasticsearch_handle_mock.return_value = es_mock
es_mock.get_mapping.return_value = no_old_mapping
SubmissionSearchStore(self.dbm, self.form_model,
self.form_model).update_store()
self.assertTrue(es_mock.put_mapping.called)
def test_should_throw_exception_when_unique_id_field_removed(self):
new_fields = [
DateField(name='q3',
code='q3',
label='Reporting date',
date_format='dd.mm.yyyy')
]
latest_form_model = FormModel(self.dbm,
"abc",
"abc",
form_code="cli001",
fields=new_fields)
with self.assertRaises(FieldTypeChangeException) as e:
SubmissionSearchStore(self.dbm, self.es, latest_form_model,
self.form_model)._verify_unique_id_change()
def test_should_add_es_mapping_when_no_existing_questions_mapping(self):
no_old_mapping = []
es_mock = MagicMock()
form_model_mock = MagicMock()
with patch("datawinners.search.submission_index.get_elasticsearch_handle") as get_elasticsearch_handle_mock:
get_elasticsearch_handle_mock.return_value = es_mock
es_mock.get_mapping.return_value = no_old_mapping
self.dbm.view.registration_form_model_by_entity_type.return_value = []
# form_model_mock.return_value = EntityFormModel(self.dbm, name='clinic',
# form_code='cli', fields=[])
SubmissionSearchStore(self.dbm, self.form_model, self.form_model).update_store()
self.assertTrue(es_mock.put_mapping.called)
def test_should_not_throw_exception_when_unique_id_field_reordered(self):
new_fields = [
UniqueIdField(unique_id_type='clinic',
name="Q1",
code="EID",
label="What is the clinic id?"),
DateField(name='q3',
code='q3',
label='Reporting date',
date_format='dd.mm.yyyy')
]
latest_form_model = FormModel(self.dbm,
"abc",
"abc",
form_code="cli001",
fields=new_fields)
SubmissionSearchStore(self.dbm, self.es, latest_form_model,
self.form_model)._verify_unique_id_change()
self.assertFalse(self.es.put_mapping.called)
def test_should_throw_exception_when_unique_id_type_field_code_changes(
self):
new_fields = [
DateField(name='q3',
code='q3',
label='Reporting date',
date_format='dd.mm.yyyy'),
UniqueIdField(unique_id_type='clinic',
name="Q1",
code="EID new",
label="What is the clinic id?")
]
latest_form_model = FormModel(self.dbm,
"abc",
"abc",
form_code="cli001",
fields=new_fields)
with self.assertRaises(FieldTypeChangeException) as e:
SubmissionSearchStore(self.dbm, self.es, latest_form_model,
self.form_model)._verify_unique_id_change()
def recreate_submissions_mapping(self, manager, questionnaire):
SubmissionSearchStore(manager, questionnaire,
None).recreate_elastic_store()
