def _enrich_questions(dbm, row, questionnaire, enrichable_questions): for question in enrichable_questions["entity_questions"]: parent = _get_parent(question, row) parent[question.code] = question.entities.get(parent.get( question.code)) or "" for question in enrichable_questions["choice_questions"]: parent = _get_parent(question, row) parent[question.code] = get_label_to_be_displayed( parent[question.code], question, questionnaire, SurveyResponseDocument._wrap_row(row)) row["doc"]["created_by"] = get_datasender_info( dbm, SurveyResponseDocument._wrap_row(row)).get('name', '') return row
def create_index(dbm, form_model, logger): form_code = form_model.form_code start_key = [form_code] end_key = [form_code, {}] rows = dbm.database.iterview("surveyresponse/surveyresponse", 1000, reduce=False, include_docs=False, startkey=start_key, endkey=end_key) es = get_elasticsearch_handle(timeout=600) survey_response_docs = [] for row in rows: survey_response = SurveyResponseDocument._wrap_row(row) search_dict = _meta_fields(survey_response, dbm) _update_with_form_model_fields(dbm, survey_response, search_dict, form_model) search_dict.update({'id': survey_response.id}) survey_response_docs.append(search_dict) if survey_response_docs: es.bulk_index(dbm.database_name, form_model.id, survey_response_docs) logger.info('Created index for survey response docs ' + str([doc.get('id') for doc in survey_response_docs]))
def process_submission(database_name, rules, row): manager = get_db_manager(database_name) success = False submission = SurveyResponseDocument._wrap_row(row) for rule in rules: success = success or rule.update_submission(submission) if success: survey_response = SurveyResponse.new_from_doc(manager, submission) survey_response.save()
def populate_submission_index(dbm, form_code=None): start_key = [form_code] if form_code else [] end_key = [form_code, {}] if form_code else [{}, {}] rows = dbm.database.iterview("surveyresponse/surveyresponse", 1000, reduce=False, include_docs=False, startkey=start_key, endkey=end_key) logger = logging.getLogger(dbm.database_name) ignored = 0 for row in rows: try: survey_response = SurveyResponseDocument._wrap_row(row) update_submission_search_index(survey_response, dbm, refresh_index=False) except FormModelDoesNotExistsException as e: ignored += 1 logger.warning(e.message) # ignore orphaned submissions On changing form code! if ignored > 0: logger.warning("Few submissions are ignored %s" % ignored)
def populate_submission_index(dbm, form_model_id=None): logger = logging.getLogger() if form_model_id is None: questionnaires = dbm.load_all_rows_in_view("surveyresponse_by_questionnaire_id", reduce=True, group=True) for q in questionnaires: logger.info('Processing questionnaire id {q}'.format(q=q.key)) populate_submission_index(dbm, q.key) else: start = time.time() rows = get_survey_responses_by_form_model_id(dbm, form_model_id) form_model = FormModel.get(dbm, form_model_id) logger = logging.getLogger(form_model.name) ignored = 0 counter = 0 error_count = 0 actions = [] es = get_elasticsearch_handle() for row in rows: try: survey_response = SurveyResponseDocument._wrap_row(row) submission_action = update_submission_search_index(survey_response, dbm, refresh_index=False, form_model=form_model, bulk=True) actions.append(submission_action) if len(actions) == settings.ES_INDEX_RECREATION_BATCH: es.bulk(actions, index=dbm.database_name, doc_type=form_model.id) actions = [] counter += 1 logger.info('No of submissions processed {counter}'.format(counter=counter)) except FormModelDoesNotExistsException as e: ignored += 1 logger.warning(e.message) # ignore orphaned submissions On changing form code! except Exception as ex: logger.exception('Exception occurred') error_count += 1 if len(actions) > 0: es.bulk(actions, index=dbm.database_name, doc_type=form_model.id) logger.warning("No of submissions ignored: {ignored}".format(ignored=ignored)) logger.warning("No of submissions had errors:{errors}".format(errors=error_count)) logger.info('Time taken (seconds) for indexing {counter} submissions of questionnaire {q} : {timetaken}' .format(counter=counter,q=form_model_id,timetaken=(time.time()-start)))
def create_submission_index(dbm, row): form_model = Project.new_from_doc(dbm, ProjectDocument.wrap(row["value"])) form_code = form_model.form_code start_key = [form_code] end_key = [form_code, {}] rows = dbm.database.iterview("surveyresponse/surveyresponse", 1000, reduce=False, include_docs=False, startkey=start_key, endkey=end_key) es = get_elasticsearch_handle(timeout=600) survey_response_docs = [] for row in rows: survey_response = SurveyResponseDocument._wrap_row(row) search_dict = _meta_fields(survey_response, dbm) _update_with_form_model_fields(dbm, survey_response, search_dict, form_model) search_dict.update({'id': survey_response.id}) survey_response_docs.append(search_dict) if survey_response_docs: es.bulk_index(dbm.database_name, form_model.id, survey_response_docs)
def add_unique_id_and_short_code_field(dbm, logger): for row in dbm.database.query(list_all_form_models, include_docs=True): try: document_data = row.doc json_data = document_data.get('json_fields') validator = None short_code_field = None short_code_dict = None index = 0 if document_data.get('is_registration_model') or document_data.get("form_code") == "delete": for index, f in enumerate(json_data): if f.get('name') == SHORT_CODE_FIELD: short_code_field = ShortCodeField(f.get('name'), f.get('code'), f.get('label'), defaultValue=f.get('defaultValue'), instruction=f.get('instruction'), required=f.get('required')) short_code_dict = f break else: for index, f in enumerate(json_data): if f.get('entity_question_flag'): start_key = [document_data.get('form_code')] end_key = [document_data.get('form_code'), {}] survey_response_rows = dbm.database.iterview("surveyresponse/surveyresponse", 1000, reduce=False, include_docs=False, startkey=start_key, endkey=end_key) if document_data.get('entity_type') != ['reporter']: short_code_field = UniqueIdField(document_data.get('entity_type')[0], f.get('name'), f.get('code'), f.get('label'), defaultValue=f.get('defaultValue'), instruction=f.get('instruction'), required=f.get('required')) validator = UniqueIdExistsValidator #Remove test field from survey responses for row in survey_response_rows: if row.get('value').get('test'): row.get('value').pop('test') else: for row in survey_response_rows: try: row.get('value').get('values').pop(f.get('code')) if row.get('value').get('test'): row.get('value').pop('test') survey_response = SurveyResponseDocument._wrap_row(row) dbm._save_document(survey_response) except Exception as e: logger.error("Survey response update failed for database %s for id %s" %(dbm.database_name,row.get('id'))) logger.error(e) short_code_dict = f break #Remove event_time flag from reporting date question elif f.get('type') == 'date' and 'event_time_field_flag' in f: f.pop('event_time_field_flag') #Remove entity type from questionnaire form models. if document_data.get('entity_type'): document_data.pop('entity_type') if short_code_dict: json_data.remove(short_code_dict) form_model = FormModel.new_from_doc(dbm, (FormModelDocument.wrap(document_data))) if short_code_field: form_model._form_fields.insert(index, short_code_field) if validator: form_model.add_validator(validator) _save_form_model_doc(dbm, form_model) except Exception as e: logger.error('Failed form model for database : %s, doc with id: %s', dbm.database_name, row.id) logger.error(e)