def _enrich_questions(dbm, row, questionnaire, enrichable_questions):
for question in enrichable_questions["entity_questions"]:
parent = _get_parent(question, row)
parent[question.code] = question.entities.get(parent.get(
question.code)) or ""
for question in enrichable_questions["choice_questions"]:
parent = _get_parent(question, row)
parent[question.code] = get_label_to_be_displayed(
parent[question.code], question, questionnaire,
SurveyResponseDocument._wrap_row(row))
row["doc"]["created_by"] = get_datasender_info(
dbm, SurveyResponseDocument._wrap_row(row)).get('name', '')
return row
def create_index(dbm, form_model, logger):
form_code = form_model.form_code
start_key = [form_code]
end_key = [form_code, {}]
rows = dbm.database.iterview("surveyresponse/surveyresponse",
1000,
reduce=False,
include_docs=False,
startkey=start_key,
endkey=end_key)
es = get_elasticsearch_handle(timeout=600)
survey_response_docs = []
for row in rows:
survey_response = SurveyResponseDocument._wrap_row(row)
search_dict = _meta_fields(survey_response, dbm)
_update_with_form_model_fields(dbm, survey_response, search_dict,
form_model)
search_dict.update({'id': survey_response.id})
survey_response_docs.append(search_dict)
if survey_response_docs:
es.bulk_index(dbm.database_name, form_model.id, survey_response_docs)
logger.info('Created index for survey response docs ' +
str([doc.get('id') for doc in survey_response_docs]))
def process_submission(database_name, rules, row):
manager = get_db_manager(database_name)
success = False
submission = SurveyResponseDocument._wrap_row(row)
for rule in rules:
success = success or rule.update_submission(submission)
if success:
survey_response = SurveyResponse.new_from_doc(manager, submission)
survey_response.save()
def populate_submission_index(dbm, form_code=None):
start_key = [form_code] if form_code else []
end_key = [form_code, {}] if form_code else [{}, {}]
rows = dbm.database.iterview("surveyresponse/surveyresponse", 1000, reduce=False, include_docs=False, startkey=start_key, endkey=end_key)
logger = logging.getLogger(dbm.database_name)
ignored = 0
for row in rows:
try:
survey_response = SurveyResponseDocument._wrap_row(row)
update_submission_search_index(survey_response, dbm, refresh_index=False)
except FormModelDoesNotExistsException as e:
ignored += 1
logger.warning(e.message) # ignore orphaned submissions On changing form code!
if ignored > 0:
logger.warning("Few submissions are ignored %s" % ignored)
def populate_submission_index(dbm, form_model_id=None):
logger = logging.getLogger()
if form_model_id is None:
questionnaires = dbm.load_all_rows_in_view("surveyresponse_by_questionnaire_id", reduce=True, group=True)
for q in questionnaires:
logger.info('Processing questionnaire id {q}'.format(q=q.key))
populate_submission_index(dbm, q.key)
else:
start = time.time()
rows = get_survey_responses_by_form_model_id(dbm, form_model_id)
form_model = FormModel.get(dbm, form_model_id)
logger = logging.getLogger(form_model.name)
ignored = 0
counter = 0
error_count = 0
actions = []
es = get_elasticsearch_handle()
for row in rows:
try:
survey_response = SurveyResponseDocument._wrap_row(row)
submission_action = update_submission_search_index(survey_response, dbm, refresh_index=False,
form_model=form_model, bulk=True)
actions.append(submission_action)
if len(actions) == settings.ES_INDEX_RECREATION_BATCH:
es.bulk(actions, index=dbm.database_name, doc_type=form_model.id)
actions = []
counter += 1
logger.info('No of submissions processed {counter}'.format(counter=counter))
except FormModelDoesNotExistsException as e:
ignored += 1
logger.warning(e.message) # ignore orphaned submissions On changing form code!
except Exception as ex:
logger.exception('Exception occurred')
error_count += 1
if len(actions) > 0:
es.bulk(actions, index=dbm.database_name, doc_type=form_model.id)
logger.warning("No of submissions ignored: {ignored}".format(ignored=ignored))
logger.warning("No of submissions had errors:{errors}".format(errors=error_count))
logger.info('Time taken (seconds) for indexing {counter} submissions of questionnaire {q} : {timetaken}'
.format(counter=counter,q=form_model_id,timetaken=(time.time()-start)))
def create_submission_index(dbm, row):
form_model = Project.new_from_doc(dbm, ProjectDocument.wrap(row["value"]))
form_code = form_model.form_code
start_key = [form_code]
end_key = [form_code, {}]
rows = dbm.database.iterview("surveyresponse/surveyresponse",
1000,
reduce=False,
include_docs=False,
startkey=start_key,
endkey=end_key)
es = get_elasticsearch_handle(timeout=600)
survey_response_docs = []
for row in rows:
survey_response = SurveyResponseDocument._wrap_row(row)
search_dict = _meta_fields(survey_response, dbm)
_update_with_form_model_fields(dbm, survey_response, search_dict,
form_model)
search_dict.update({'id': survey_response.id})
survey_response_docs.append(search_dict)
if survey_response_docs:
es.bulk_index(dbm.database_name, form_model.id, survey_response_docs)
def add_unique_id_and_short_code_field(dbm, logger):
for row in dbm.database.query(list_all_form_models, include_docs=True):
try:
document_data = row.doc
json_data = document_data.get('json_fields')
validator = None
short_code_field = None
short_code_dict = None
index = 0
if document_data.get('is_registration_model') or document_data.get("form_code") == "delete":
for index, f in enumerate(json_data):
if f.get('name') == SHORT_CODE_FIELD:
short_code_field = ShortCodeField(f.get('name'), f.get('code'), f.get('label'),
defaultValue=f.get('defaultValue'),
instruction=f.get('instruction'), required=f.get('required'))
short_code_dict = f
break
else:
for index, f in enumerate(json_data):
if f.get('entity_question_flag'):
start_key = [document_data.get('form_code')]
end_key = [document_data.get('form_code'), {}]
survey_response_rows = dbm.database.iterview("surveyresponse/surveyresponse", 1000, reduce=False, include_docs=False, startkey=start_key, endkey=end_key)
if document_data.get('entity_type') != ['reporter']:
short_code_field = UniqueIdField(document_data.get('entity_type')[0], f.get('name'),
f.get('code'),
f.get('label'), defaultValue=f.get('defaultValue'),
instruction=f.get('instruction'),
required=f.get('required'))
validator = UniqueIdExistsValidator
#Remove test field from survey responses
for row in survey_response_rows:
if row.get('value').get('test'):
row.get('value').pop('test')
else:
for row in survey_response_rows:
try:
row.get('value').get('values').pop(f.get('code'))
if row.get('value').get('test'):
row.get('value').pop('test')
survey_response = SurveyResponseDocument._wrap_row(row)
dbm._save_document(survey_response)
except Exception as e:
logger.error("Survey response update failed for database %s for id %s" %(dbm.database_name,row.get('id')))
logger.error(e)
short_code_dict = f
break
#Remove event_time flag from reporting date question
elif f.get('type') == 'date' and 'event_time_field_flag' in f:
f.pop('event_time_field_flag')
#Remove entity type from questionnaire form models.
if document_data.get('entity_type'):
document_data.pop('entity_type')
if short_code_dict:
json_data.remove(short_code_dict)
form_model = FormModel.new_from_doc(dbm, (FormModelDocument.wrap(document_data)))
if short_code_field:
form_model._form_fields.insert(index, short_code_field)
if validator:
form_model.add_validator(validator)
_save_form_model_doc(dbm, form_model)
except Exception as e:
logger.error('Failed form model for database : %s, doc with id: %s', dbm.database_name,
row.id)
logger.error(e)
