def get_list_of_indexes_to_reindex(full_reindex=False): db_names = all_db_names() try: list_of_indexes_out_of_sync = [] total_submissions = 0 for database_name in db_names: dbm = get_db_manager(database_name) questionnaires = dbm.load_all_rows_in_view('questionnaire') if not questionnaires: continue for row in questionnaires: if row['value']['is_registration_model']: continue form_model_doc = FormModelDocument.wrap(row["value"]) if full_reindex or is_mapping_out_of_sync(form_model_doc, dbm): es = Elasticsearch(hosts=[{ "host": ELASTIC_SEARCH_HOST, "port": ELASTIC_SEARCH_PORT }]) search = Search(using=es, index=dbm.database_name, doc_type=form_model_doc.id) no_of_submissions = search.count() questionnaire_info = dict( db_name=database_name, questionnaire_id=form_model_doc.id, name=form_model_doc.name, no_of_submissions=no_of_submissions) total_submissions += no_of_submissions list_of_indexes_out_of_sync.append(questionnaire_info) return list_of_indexes_out_of_sync, total_submissions except Exception as e: pass
def convert_field_code_to_lower_case(db_name): logger = logging.getLogger(db_name) try: dbm = get_db_manager(db_name) rows = dbm.load_all_rows_in_view('questionnaire', reduce=False) for row in rows: form_model = FormModel.new_from_doc( dbm, FormModelDocument.wrap(row['value'])) is_upper = False for field in form_model.fields: if re.match(r".*[A-Z]+.*", field.code): logger.info("doc id: %s, field code: %s", form_model.id, field.code) is_upper = True field._dict['code'] = field.code.lower() if is_upper: form_model.save() survey_responses = survey_responses_by_form_code( dbm, form_model.form_code) for survey_response in survey_responses: convert_dict_keys_to_lowercase(survey_response.values) survey_response.save() logger.info("Modified survey response id: %s" % survey_response.uuid) except Exception as e: logger.exception(e.message) mark_as_completed(db_name)
def get_all_form_models(dbm, data_sender_id=None): questionnaires = [] if data_sender_id: rows = dbm.load_all_rows_in_view('projects_by_datasenders', startkey=data_sender_id, endkey=data_sender_id, include_docs=True) idnr_questionnaires = [] for row in rows: row.update({'value': row["doc"]}) subject_docs = get_subject_form_model_docs_of_questionnaire(dbm, FormModelDocument.wrap(row['doc'])) duplicate_docs = [] for subject_doc in subject_docs: for questionnaire in idnr_questionnaires: if subject_doc.id == questionnaire.id: duplicate_docs.append(subject_doc) for duplicate_doc in duplicate_docs: if duplicate_doc in subject_docs: subject_docs.remove(duplicate_doc) idnr_questionnaires.extend(subject_docs) rows.extend(idnr_questionnaires) else: rows = dbm.load_all_rows_in_view('all_questionnaire') for row in rows: if 'is_poll' not in row['value'] or row['value']['is_poll'] is False: questionnaires.append(row) return questionnaires
def get_form_model_by_code(dbm, code): row_value = get_form_model_document(code, dbm) if row_value.get('is_registration_model') or row_value.get( 'form_code') == ENTITY_DELETION_FORM_CODE: return EntityFormModel.new_from_doc( dbm, EntityFormModelDocument.wrap(row_value)) return FormModel.new_from_doc(dbm, FormModelDocument.wrap(row_value))
def _get_form_models(dbm, form_code): form_model_docs = _get_form_model_docs(dbm, form_code) if form_model_docs: return [ FormModel.new_from_doc(dbm, FormModelDocument.wrap(form_model_doc)) for form_model_doc in form_model_docs ] return None
def create_all_mappings(dbm): logger = logging.getLogger(dbm.database_name) for row in dbm.load_all_rows_in_view('questionnaire'): try: if row['value']['is_registration_model']: entity_form_model_change_handler(EntityFormModelDocument.wrap(row["value"]), dbm) else: form_model_change_handler(FormModelDocument.wrap(row["value"]), dbm) except Exception as e: logger.exception(e.message)
def get_subject_form_models(manager): form_models = {} form_model_values = manager.load_all_rows_in_view('questionnaire') for each in form_model_values: form_model = FormModel.new_from_doc( manager, FormModelDocument.wrap(each['value'])) if form_model.is_registration_form( ) and not form_model.entity_defaults_to_reporter(): form_models[form_model.entity_type[0]] = form_model return form_models
def registration_form_model_codes(dbm): rows = dbm.view.registration_form_model_by_entity_type(include_docs=True) # Form code for Data sender registration form_codes = ['reg'] if len(rows): for row in rows: doc = FormModelDocument.wrap(row['doc']) form_model = FormModel.new_from_doc(dbm, doc) form_codes.append(form_model.form_code) return form_codes
def update_mapping(database_name): try: dbm = get_db_manager(database_name) logger = logging.getLogger(database_name) for row in dbm.load_all_rows_in_view('questionnaire'): form_model_doc = FormModelDocument.wrap(row["value"]) form_model_change_handler(form_model_doc, dbm) except Exception as e: logger.error( "Mapping update failed for database %s for form model %s " % (database_name, form_model_doc.form_code)) logger.error(e) mark_as_completed(database_name)
def migrate_subject_form_code_to_add_regex_validator_in_short_code(db_name): logger = logging.getLogger(db_name) logger.info('Starting Migration') mark_as_completed(db_name) manager = get_db_manager(db_name) for row in manager.database.query(map_subject_form_model): try: doc = FormModelDocument.wrap(row['value']) form_model = FormModel.new_from_doc(manager, doc) add_regex_constraint_to_short_code(form_model, logger) except Exception as e: logger.exception("FAILED to migrate:%s " % row['value']['_id']) logger.info('Completed Migration')
def delete_language_setting_for_form_models(dbm): assert isinstance(dbm, DatabaseManager) rows = dbm.load_all_rows_in_view('questionnaire') for row in rows: try: form_model_row_value = row['value'] form_model_row_value = delete_language_redundant_settings( form_model_row_value) doc = FormModelDocument.wrap(form_model_row_value) form_model = FormModel.new_from_doc(dbm, doc) form_model.save() except Exception as e: print( "*******************form model error *************************************", row['key']) print e print "*************************************************"
def create_submission_index(database_name, logger): dbm = get_db_manager(database_name) for row in dbm.load_all_rows_in_view('questionnaire'): try: form_model = FormModel.new_from_doc( dbm, FormModelDocument.wrap(row["value"])) except FormModelDoesNotExistsException as e: logger.exception(e.message) continue if form_model.is_entity_registration_form( ) or "delete" == form_model.form_code: continue try: es.delete_all(dbm.database_name, form_model.id) except Exception as ignore: logger.info("Error while deleting index %s" % ignore.message) create_submission_mapping(dbm, form_model) create_index(dbm, form_model, logger)
def is_authorized_for_questionnaire(dbm, request_user, form_code): try: user_profile = request_user.get_profile() if not user_profile.reporter: return True questionnaire = get_form_model_by_code(dbm, form_code) if isinstance(questionnaire, EntityFormModel): rows = dbm.view.projects_by_subject_type(include_docs=True) for row in rows: fm = FormModel.new_from_doc(dbm, FormModelDocument.wrap(row['doc'])) if not fm.is_void( ) and user_profile.reporter_id in fm.data_senders: return True if questionnaire.is_void( ) or user_profile.reporter_id not in questionnaire.data_senders: return False except FormModelDoesNotExistsException as e: return False return True
def create_all_mappings(dbm): for row in dbm.load_all_rows_in_view('questionnaire'): if row['value']['is_registration_model']: entity_form_model_change_handler(EntityFormModelDocument.wrap(row["value"]), dbm) else: form_model_change_handler(FormModelDocument.wrap(row["value"]), dbm)
def _row_to_form_model(row): doc = FormModelDocument.wrap(row['value']) return FormModel.new_from_doc(dbm, doc)
def get_form_model(manager, raw_str): doc = FormModelDocument.wrap(raw_str) form_model = FormModel.new_from_doc(manager, doc) return form_model
def get_form_model(manager, questionnaire): doc = FormModelDocument.wrap(questionnaire['value']) form_model = FormModel.new_from_doc(manager, doc) return form_model
def add_unique_id_and_short_code_field(dbm, logger): for row in dbm.database.query(list_all_form_models, include_docs=True): try: document_data = row.doc json_data = document_data.get('json_fields') validator = None short_code_field = None short_code_dict = None index = 0 if document_data.get('is_registration_model') or document_data.get("form_code") == "delete": for index, f in enumerate(json_data): if f.get('name') == SHORT_CODE_FIELD: short_code_field = ShortCodeField(f.get('name'), f.get('code'), f.get('label'), defaultValue=f.get('defaultValue'), instruction=f.get('instruction'), required=f.get('required')) short_code_dict = f break else: for index, f in enumerate(json_data): if f.get('entity_question_flag'): start_key = [document_data.get('form_code')] end_key = [document_data.get('form_code'), {}] survey_response_rows = dbm.database.iterview("surveyresponse/surveyresponse", 1000, reduce=False, include_docs=False, startkey=start_key, endkey=end_key) if document_data.get('entity_type') != ['reporter']: short_code_field = UniqueIdField(document_data.get('entity_type')[0], f.get('name'), f.get('code'), f.get('label'), defaultValue=f.get('defaultValue'), instruction=f.get('instruction'), required=f.get('required')) validator = UniqueIdExistsValidator #Remove test field from survey responses for row in survey_response_rows: if row.get('value').get('test'): row.get('value').pop('test') else: for row in survey_response_rows: try: row.get('value').get('values').pop(f.get('code')) if row.get('value').get('test'): row.get('value').pop('test') survey_response = SurveyResponseDocument._wrap_row(row) dbm._save_document(survey_response) except Exception as e: logger.error("Survey response update failed for database %s for id %s" %(dbm.database_name,row.get('id'))) logger.error(e) short_code_dict = f break #Remove event_time flag from reporting date question elif f.get('type') == 'date' and 'event_time_field_flag' in f: f.pop('event_time_field_flag') #Remove entity type from questionnaire form models. if document_data.get('entity_type'): document_data.pop('entity_type') if short_code_dict: json_data.remove(short_code_dict) form_model = FormModel.new_from_doc(dbm, (FormModelDocument.wrap(document_data))) if short_code_field: form_model._form_fields.insert(index, short_code_field) if validator: form_model.add_validator(validator) _save_form_model_doc(dbm, form_model) except Exception as e: logger.error('Failed form model for database : %s, doc with id: %s', dbm.database_name, row.id) logger.error(e)
def create_all_mappings(dbm): for row in dbm.load_all_rows_in_view('questionnaire'): form_model_doc = FormModelDocument.wrap(row["value"]) form_model_change_handler(form_model_doc, dbm)