def register_postsave_handlers():
global _postsave_registered
if _postsave_registered: return
EntityDocument.register_post_update(entity_search_update)
ContactDocument.register_post_update(contact_search_update)
EntityFormModelDocument.register_post_update(entity_form_model_change_handler)
FormModelDocument.register_post_update(form_model_change_handler)
FormModelDocument.register_post_update(update_datasender_for_project_change)
SurveyResponseDocument.register_post_update(update_submission_search_index)
_postsave_registered = True
def __init__(self, dbm, name=None, label=None, form_code=None, fields=None, entity_type=None, type=None,
language="eng", state=attributes.ACTIVE_STATE):
assert isinstance(dbm, DatabaseManager)
assert name is None or is_not_empty(name)
assert fields is None or is_sequence(fields)
assert form_code is None or (is_string(form_code) and is_not_empty(form_code))
assert type is None or is_not_empty(type)
assert entity_type is None or is_sequence(entity_type)
DataObject.__init__(self, dbm)
self._form_fields = []
self.errors = []
# Are we being constructed from scratch or existing doc?
if name is None:
return
# Not made from existing doc, so build ourselves up
self._form_fields = fields
self.validate_fields()
doc = FormModelDocument()
doc.name = name
doc.add_label(language, label)
doc.form_code = form_code
doc.entity_type = entity_type
doc.type = type
doc.state = state
doc.active_languages = language
DataObject._set_document(self, doc)
def get_all_form_models(dbm, data_sender_id=None):
questionnaires = []
if data_sender_id:
rows = dbm.load_all_rows_in_view('projects_by_datasenders', startkey=data_sender_id, endkey=data_sender_id,
include_docs=True)
idnr_questionnaires = []
for row in rows:
row.update({'value': row["doc"]})
subject_docs = get_subject_form_model_docs_of_questionnaire(dbm, FormModelDocument.wrap(row['doc']))
duplicate_docs = []
for subject_doc in subject_docs:
for questionnaire in idnr_questionnaires:
if subject_doc.id == questionnaire.id:
duplicate_docs.append(subject_doc)
for duplicate_doc in duplicate_docs:
if duplicate_doc in subject_docs:
subject_docs.remove(duplicate_doc)
idnr_questionnaires.extend(subject_docs)
rows.extend(idnr_questionnaires)
else:
rows = dbm.load_all_rows_in_view('all_questionnaire')
for row in rows:
if 'is_poll' not in row['value'] or row['value']['is_poll'] is False:
questionnaires.append(row)
return questionnaires
def get_list_of_indexes_to_reindex(full_reindex=False):
db_names = all_db_names()
try:
list_of_indexes_out_of_sync = []
total_submissions = 0
for database_name in db_names:
dbm = get_db_manager(database_name)
questionnaires = dbm.load_all_rows_in_view('questionnaire')
if not questionnaires:
continue
for row in questionnaires:
if row['value']['is_registration_model']:
continue
form_model_doc = FormModelDocument.wrap(row["value"])
if full_reindex or is_mapping_out_of_sync(form_model_doc, dbm):
es = Elasticsearch(hosts=[{
"host": ELASTIC_SEARCH_HOST,
"port": ELASTIC_SEARCH_PORT
}])
search = Search(using=es,
index=dbm.database_name,
doc_type=form_model_doc.id)
no_of_submissions = search.count()
questionnaire_info = dict(
db_name=database_name,
questionnaire_id=form_model_doc.id,
name=form_model_doc.name,
no_of_submissions=no_of_submissions)
total_submissions += no_of_submissions
list_of_indexes_out_of_sync.append(questionnaire_info)
return list_of_indexes_out_of_sync, total_submissions
except Exception as e:
pass
def convert_field_code_to_lower_case(db_name):
logger = logging.getLogger(db_name)
try:
dbm = get_db_manager(db_name)
rows = dbm.load_all_rows_in_view('questionnaire', reduce=False)
for row in rows:
form_model = FormModel.new_from_doc(
dbm, FormModelDocument.wrap(row['value']))
is_upper = False
for field in form_model.fields:
if re.match(r".*[A-Z]+.*", field.code):
logger.info("doc id: %s, field code: %s", form_model.id,
field.code)
is_upper = True
field._dict['code'] = field.code.lower()
if is_upper:
form_model.save()
survey_responses = survey_responses_by_form_code(
dbm, form_model.form_code)
for survey_response in survey_responses:
convert_dict_keys_to_lowercase(survey_response.values)
survey_response.save()
logger.info("Modified survey response id: %s" %
survey_response.uuid)
except Exception as e:
logger.exception(e.message)
mark_as_completed(db_name)
def _set_doc(self, form_code, is_registration_model, label, language,
name):
doc = FormModelDocument()
doc.name = name
doc.set_label(label)
doc.form_code = form_code
doc.active_languages = [language]
doc.is_registration_model = is_registration_model
DataObject._set_document(self, doc)
def _get_form_models(dbm, form_code):
form_model_docs = _get_form_model_docs(dbm, form_code)
if form_model_docs:
return [
FormModel.new_from_doc(dbm, FormModelDocument.wrap(form_model_doc))
for form_model_doc in form_model_docs
]
return None
def get_form_model_by_code(dbm, code):
row_value = get_form_model_document(code, dbm)
if row_value.get('is_registration_model') or row_value.get(
'form_code') == ENTITY_DELETION_FORM_CODE:
return EntityFormModel.new_from_doc(
dbm, EntityFormModelDocument.wrap(row_value))
return FormModel.new_from_doc(dbm, FormModelDocument.wrap(row_value))
def get_form_model_doc(self, fields=[]):
if len(fields) == 0:
fields = [{
"constraints": [('range', {
"max": 10,
"min": 0
})],
"label": "",
"type": "integer",
"name": "What is your age?",
"code": "AGE",
"required": False
}, {
"choices": [{
"text": "Pune"
}, {
"text": "Bangalore"
}],
"label": "",
"type": "select",
"name": "Where do you live?",
"code": "PLC",
"required": False
}]
document = FormModelDocument()
document.json_fields = fields
document.document_type = "FormModel"
document.form_code = "F1"
document.name = "New Project"
document.type = "survey"
document.type = "survey"
return document
def create_all_mappings(dbm):
logger = logging.getLogger(dbm.database_name)
for row in dbm.load_all_rows_in_view('questionnaire'):
try:
if row['value']['is_registration_model']:
entity_form_model_change_handler(EntityFormModelDocument.wrap(row["value"]), dbm)
else:
form_model_change_handler(FormModelDocument.wrap(row["value"]), dbm)
except Exception as e:
logger.exception(e.message)
def get_subject_form_models(manager):
form_models = {}
form_model_values = manager.load_all_rows_in_view('questionnaire')
for each in form_model_values:
form_model = FormModel.new_from_doc(
manager, FormModelDocument.wrap(each['value']))
if form_model.is_registration_form(
) and not form_model.entity_defaults_to_reporter():
form_models[form_model.entity_type[0]] = form_model
return form_models
def registration_form_model_codes(dbm):
rows = dbm.view.registration_form_model_by_entity_type(include_docs=True)
# Form code for Data sender registration
form_codes = ['reg']
if len(rows):
for row in rows:
doc = FormModelDocument.wrap(row['doc'])
form_model = FormModel.new_from_doc(dbm, doc)
form_codes.append(form_model.form_code)
return form_codes
def __init__(self,
dbm,
name=None,
label=None,
form_code=None,
fields=None,
entity_type=None,
type=None,
language="eng",
state=attributes.ACTIVE_STATE):
assert isinstance(dbm, DatabaseManager)
assert name is None or is_not_empty(name)
assert fields is None or is_sequence(fields)
assert form_code is None or (is_string(form_code)
and is_not_empty(form_code))
assert type is None or is_not_empty(type)
assert entity_type is None or is_sequence(entity_type)
DataObject.__init__(self, dbm)
self._form_fields = []
self.errors = []
# Are we being constructed from scratch or existing doc?
if name is None:
return
# Not made from existing doc, so build ourselves up
self._form_fields = fields
self.validate_fields()
doc = FormModelDocument()
doc.name = name
doc.add_label(language, label)
doc.form_code = form_code
doc.entity_type = entity_type
doc.type = type
doc.state = state
doc.active_languages = language
DataObject._set_document(self, doc)
def migrate_subject_form_code_to_add_regex_validator_in_short_code(db_name):
logger = logging.getLogger(db_name)
logger.info('Starting Migration')
mark_as_completed(db_name)
manager = get_db_manager(db_name)
for row in manager.database.query(map_subject_form_model):
try:
doc = FormModelDocument.wrap(row['value'])
form_model = FormModel.new_from_doc(manager, doc)
add_regex_constraint_to_short_code(form_model, logger)
except Exception as e:
logger.exception("FAILED to migrate:%s " % row['value']['_id'])
logger.info('Completed Migration')
def update_mapping(database_name):
try:
dbm = get_db_manager(database_name)
logger = logging.getLogger(database_name)
for row in dbm.load_all_rows_in_view('questionnaire'):
form_model_doc = FormModelDocument.wrap(row["value"])
form_model_change_handler(form_model_doc, dbm)
except Exception as e:
logger.error(
"Mapping update failed for database %s for form model %s " %
(database_name, form_model_doc.form_code))
logger.error(e)
mark_as_completed(database_name)
def delete_language_setting_for_form_models(dbm):
assert isinstance(dbm, DatabaseManager)
rows = dbm.load_all_rows_in_view('questionnaire')
for row in rows:
try:
form_model_row_value = row['value']
form_model_row_value = delete_language_redundant_settings(
form_model_row_value)
doc = FormModelDocument.wrap(form_model_row_value)
form_model = FormModel.new_from_doc(dbm, doc)
form_model.save()
except Exception as e:
print(
"*******************form model error *************************************",
row['key'])
print e
print "*************************************************"
def create_submission_index(database_name, logger):
dbm = get_db_manager(database_name)
for row in dbm.load_all_rows_in_view('questionnaire'):
try:
form_model = FormModel.new_from_doc(
dbm, FormModelDocument.wrap(row["value"]))
except FormModelDoesNotExistsException as e:
logger.exception(e.message)
continue
if form_model.is_entity_registration_form(
) or "delete" == form_model.form_code:
continue
try:
es.delete_all(dbm.database_name, form_model.id)
except Exception as ignore:
logger.info("Error while deleting index %s" % ignore.message)
create_submission_mapping(dbm, form_model)
create_index(dbm, form_model, logger)
def is_authorized_for_questionnaire(dbm, request_user, form_code):
try:
user_profile = request_user.get_profile()
if not user_profile.reporter:
return True
questionnaire = get_form_model_by_code(dbm, form_code)
if isinstance(questionnaire, EntityFormModel):
rows = dbm.view.projects_by_subject_type(include_docs=True)
for row in rows:
fm = FormModel.new_from_doc(dbm,
FormModelDocument.wrap(row['doc']))
if not fm.is_void(
) and user_profile.reporter_id in fm.data_senders:
return True
if questionnaire.is_void(
) or user_profile.reporter_id not in questionnaire.data_senders:
return False
except FormModelDoesNotExistsException as e:
return False
return True
def create_all_mappings(dbm):
for row in dbm.load_all_rows_in_view('questionnaire'):
if row['value']['is_registration_model']:
entity_form_model_change_handler(EntityFormModelDocument.wrap(row["value"]), dbm)
else:
form_model_change_handler(FormModelDocument.wrap(row["value"]), dbm)
def _row_to_form_model(row):
doc = FormModelDocument.wrap(row['value'])
return FormModel.new_from_doc(dbm, doc)
def add_unique_id_and_short_code_field(dbm, logger):
for row in dbm.database.query(list_all_form_models, include_docs=True):
try:
document_data = row.doc
json_data = document_data.get('json_fields')
validator = None
short_code_field = None
short_code_dict = None
index = 0
if document_data.get('is_registration_model') or document_data.get("form_code") == "delete":
for index, f in enumerate(json_data):
if f.get('name') == SHORT_CODE_FIELD:
short_code_field = ShortCodeField(f.get('name'), f.get('code'), f.get('label'),
defaultValue=f.get('defaultValue'),
instruction=f.get('instruction'), required=f.get('required'))
short_code_dict = f
break
else:
for index, f in enumerate(json_data):
if f.get('entity_question_flag'):
start_key = [document_data.get('form_code')]
end_key = [document_data.get('form_code'), {}]
survey_response_rows = dbm.database.iterview("surveyresponse/surveyresponse", 1000, reduce=False, include_docs=False, startkey=start_key, endkey=end_key)
if document_data.get('entity_type') != ['reporter']:
short_code_field = UniqueIdField(document_data.get('entity_type')[0], f.get('name'),
f.get('code'),
f.get('label'), defaultValue=f.get('defaultValue'),
instruction=f.get('instruction'),
required=f.get('required'))
validator = UniqueIdExistsValidator
#Remove test field from survey responses
for row in survey_response_rows:
if row.get('value').get('test'):
row.get('value').pop('test')
else:
for row in survey_response_rows:
try:
row.get('value').get('values').pop(f.get('code'))
if row.get('value').get('test'):
row.get('value').pop('test')
survey_response = SurveyResponseDocument._wrap_row(row)
dbm._save_document(survey_response)
except Exception as e:
logger.error("Survey response update failed for database %s for id %s" %(dbm.database_name,row.get('id')))
logger.error(e)
short_code_dict = f
break
#Remove event_time flag from reporting date question
elif f.get('type') == 'date' and 'event_time_field_flag' in f:
f.pop('event_time_field_flag')
#Remove entity type from questionnaire form models.
if document_data.get('entity_type'):
document_data.pop('entity_type')
if short_code_dict:
json_data.remove(short_code_dict)
form_model = FormModel.new_from_doc(dbm, (FormModelDocument.wrap(document_data)))
if short_code_field:
form_model._form_fields.insert(index, short_code_field)
if validator:
form_model.add_validator(validator)
_save_form_model_doc(dbm, form_model)
except Exception as e:
logger.error('Failed form model for database : %s, doc with id: %s', dbm.database_name,
row.id)
logger.error(e)
def get_form_model(manager, raw_str):
doc = FormModelDocument.wrap(raw_str)
form_model = FormModel.new_from_doc(manager, doc)
return form_model
def test_should_create_a_questionnaire_from_dictionary(self):
fields = [
{
"name": "What are you reporting on?",
"defaultValue": "",
"label": {
"eng": "Entity being reported on"
},
"entity_question_flag": True,
"type": "text",
"ddtype": self.default_ddtype.to_json(),
"code": "eid",
"length": {"min": 1, "max": 10},
},
{
"range": {
"max": 10,
"min": 0
},
"label": {"eng": ""},
"type": "integer",
"ddtype": self.default_ddtype.to_json(),
"name": "What is your age?",
"code": "AGE"
},
{
"choices": [
{
"text": {"eng": "Pune"}
},
{
"text": {"eng": "Bangalore"}
}
],
"label": {"eng": ""},
"type": "select",
"ddtype": self.default_ddtype.to_json(),
"name": "Where do you live?",
"code": "PLC"
}]
document = FormModelDocument()
document.json_fields = fields
document.entity_type = ["Reporter"]
document.document_type = "FormModel"
document.form_code = "F1"
document.name = "New Project"
document.type = "survey"
document.type = "survey"
entityQ = TextField(name="What are you reporting on?", code="eid",
label="Entity being reported on", entity_question_flag=True,
length=TextConstraint(min=1, max=10), ddtype=self.default_ddtype)
ageQ = IntegerField(name="What is your age?", code="AGE", label="",
range=NumericConstraint(min=0, max=10), ddtype=self.default_ddtype)
placeQ = SelectField(name="Where do you live?", code="PLC", label="",
options=[{"text": {"eng": "Pune"}}, {"text": {"eng": "Bangalore"}}],
single_select_flag=False, ddtype=self.default_ddtype)
questions = [entityQ, ageQ, placeQ]
questionnaire = FormModel.new_from_doc(self.dbm, document)
self.maxDiff = None
self.assertListEqual(questionnaire.entity_type, ["Reporter"])
self.assertEqual(questionnaire.name, "New Project")
self.assertEqual(questionnaire.type, "survey")
for i in range(len(questions)):
self.assertEqual(questionnaire.fields[i]._to_json(), questions[i]._to_json())
def test_should_set_entity_type_in_doc(self):
entity_form_model = EntityFormModel(self.manager)
entity_form_model._doc = FormModelDocument()
entity_form_model.entity_type = ["WaterPoint", "Dam"]
self.assertEqual(entity_form_model.entity_type, ["WaterPoint", "Dam"])
def get_form_model(manager, questionnaire):
doc = FormModelDocument.wrap(questionnaire['value'])
form_model = FormModel.new_from_doc(manager, doc)
return form_model
def test_should_create_a_questionnaire_from_dictionary(self):
fields = [{
"name": "What are you reporting on?",
"defaultValue": "",
"label": {
"eng": "Entity being reported on"
},
"entity_question_flag": True,
"type": "text",
"ddtype": self.default_ddtype.to_json(),
"code": "eid",
"length": {
"min": 1,
"max": 10
},
}, {
"range": {
"max": 10,
"min": 0
},
"label": {
"eng": ""
},
"type": "integer",
"ddtype": self.default_ddtype.to_json(),
"name": "What is your age?",
"code": "AGE"
}, {
"choices": [{
"text": {
"eng": "Pune"
}
}, {
"text": {
"eng": "Bangalore"
}
}],
"label": {
"eng": ""
},
"type":
"select",
"ddtype":
self.default_ddtype.to_json(),
"name":
"Where do you live?",
"code":
"PLC"
}]
document = FormModelDocument()
document.json_fields = fields
document.entity_type = ["Reporter"]
document.document_type = "FormModel"
document.form_code = "F1"
document.name = "New Project"
document.type = "survey"
document.type = "survey"
entityQ = TextField(name="What are you reporting on?",
code="eid",
label="Entity being reported on",
entity_question_flag=True,
length=TextConstraint(min=1, max=10),
ddtype=self.default_ddtype)
ageQ = IntegerField(name="What is your age?",
code="AGE",
label="",
range=NumericConstraint(min=0, max=10),
ddtype=self.default_ddtype)
placeQ = SelectField(name="Where do you live?",
code="PLC",
label="",
options=[{
"text": {
"eng": "Pune"
}
}, {
"text": {
"eng": "Bangalore"
}
}],
single_select_flag=False,
ddtype=self.default_ddtype)
questions = [entityQ, ageQ, placeQ]
questionnaire = FormModel.new_from_doc(self.dbm, document)
self.maxDiff = None
self.assertListEqual(questionnaire.entity_type, ["Reporter"])
self.assertEqual(questionnaire.name, "New Project")
self.assertEqual(questionnaire.type, "survey")
for i in range(len(questions)):
self.assertEqual(questionnaire.fields[i]._to_json(),
questions[i]._to_json())
def create_all_mappings(dbm):
for row in dbm.load_all_rows_in_view('questionnaire'):
form_model_doc = FormModelDocument.wrap(row["value"])
form_model_change_handler(form_model_doc, dbm)