def review_and_test(request, project_id=None):
manager = get_database_manager(request.user)
project = Project.load(manager.database, project_id)
form_model = FormModel.get(manager, project.qid)
if request.method == 'GET':
number_of_registered_subjects = get_non_voided_entity_count_for_type(manager, project.entity_type)
number_of_registered_data_senders = len(project.data_senders)
fields = form_model.fields
if form_model.is_entity_type_reporter():
fields = helper.hide_entity_question(form_model.fields)
is_reminder = "enabled" if len(Reminder.objects.filter(project_id=project.id)) != 0 else "disabled"
project_devices = project.devices
devices = ", ".join(project_devices).replace('sms', 'SMS').replace('web', 'Web').replace('smartPhone',
'Smartphone')
in_trial_mode = _in_trial_mode(request)
return render_to_response('project/review_and_test.html', {'project': project, 'fields': fields,
'project_links': make_project_links(project,
form_model.form_code),
'is_quota_reached': is_quota_reached(request),
'number_of_datasenders': number_of_registered_data_senders
,
'number_of_subjects': number_of_registered_subjects,
"is_reminder": is_reminder,
"in_trial_mode": in_trial_mode,
"devices": devices},
context_instance=RequestContext(request))
def test_should_add_select1_field(self):
select_question = FormModel.get(self.manager,
self.form_model_id).fields[3]
option_constraint = select_question.options
self.assertEquals(len(option_constraint), 2)
self.assertEquals(option_constraint[0].get("val"), 'a')
def test_should_add_snapshot_when_modifying(self):
original_form = self.form_model
original_form.create_snapshot()
original_form.save()
updated_form = FormModel.get(self.manager, self.form_model_id)
self.assertTrue(len(updated_form.snapshots) == 1)
def test_get_form_model(self):
form = FormModel.get(self.manager, self.form_model_id)
self.assertIsNotNone(form.id)
constraints = form.fields[1].constraints
self.assertEqual(10, constraints[0].max)
self.assertEqual(5, constraints[0].min)
self.assertEqual("\w+", constraints[1].pattern)
def merge_project_and_form_model_for(dbm, logger):
for row in dbm.database.query(list_all_projects, include_docs=True):
try:
project_data = row.doc
form_model = FormModel.get(dbm, project_data.get("qid"))
form_model_doc = form_model._doc
form_model_doc['goals'] = project_data['goals']
form_model_doc['name'] = project_data['name']
form_model_doc['devices'] = project_data['devices']
form_model_doc['data_senders'] = project_data['data_senders']
form_model_doc['reminder_and_deadline'] = project_data[
'reminder_and_deadline']
form_model_doc['sender_group'] = project_data['sender_group']
try:
del form_model_doc['state']
except KeyError as e:
logger.warn(e)
dbm._save_document(form_model_doc)
update_reminders(dbm, project_data, logger)
logger.info("Deleting project with id: %s", row.id)
dbm.database.delete(row.doc)
except Exception as e:
logger.error(
'Merging project and form_model failed for database : %s, project_doc with id: %s',
dbm.database_name, row.id)
logger.error(e)
def _build_fixtures(self):
entity_type = ["clinic"]
fields = []
fields.append(IntegerField('beds', 'BEDS', 'beds label'))
fields.append(IntegerField('meds', 'MEDS', 'meds label'))
fields.append(TextField('doctor', 'DOCTOR', 'doctor label'))
fields.append(ShortCodeField('clinic', 'ID', 'clinic label'))
try:
self.form_model = EntityFormModel(self.manager,
entity_type=entity_type,
name='form_model_name',
label='form_model_label',
form_code='clf12',
type='form_model_type',
fields=fields,
is_registration_model=True)
form_model_id = self.form_model.save()
self.form_model = FormModel.get(self.manager, form_model_id)
except DataObjectAlreadyExists:
pass
[
EntityBuilder(self.manager, entity_type,
'cl00%d' % i).build().save() for i in range(1, 6)
]
def get_questions_paginated_or_by_ids(request):
manager = get_database_manager(request.user)
start = int(request.GET.get('start', '0'))
length = int(request.GET.get('length', '10'))
ids = request.GET.getlist('ids')
if ids:
projects = [
_project_details(manager, project_id) for project_id in ids
]
projects = list(filter(lambda x: x != None, projects))
return response_json_cors(projects)
project_list = []
rows = manager.load_all_rows_in_view('all_projects', descending=True)
for row in rows:
questionnaire = FormModel.get(manager, row['id'])
if questionnaire.xform:
project_temp = dict(name=questionnaire.name,
project_uuid=questionnaire.id,
version=questionnaire._doc.rev)
project_list.append(project_temp)
return response_json_cors({
"projects": project_list[start:start + length],
"total": len(project_list),
"start": start,
"length": length
})
def get_projects_status(request):
response_projects = []
manager = get_database_manager(request.user)
client_projects = json.loads(request.POST['projects'])
for client_project in client_projects:
try:
server_project = FormModel.get(manager, client_project['id'])
if (server_project._doc.void):
response_projects.appened({
'id': client_project['id'],
'status': 'server-deleted'
})
elif server_project.revision != client_project['rev']:
response_projects.append({
'id': server_project.id,
'status': 'outdated'
})
except Exception:
response_projects.append({
'id': client_project['id'],
'status': 'server-deleted'
})
return response_json_cors(response_projects)
def delete_project(manager, project, void=True):
project_id, qid = project.id, project.qid
[reminder.void(void) for reminder in (Reminder.objects.filter(project_id=project_id))]
questionnaire = FormModel.get(manager, qid)
[submission.void(void) for submission in get_submissions(manager, questionnaire.form_code, None, None)]
questionnaire.void(void)
project.set_void(manager, void)
def xform_for(dbm, form_id, reporter_id):
questionnaire = FormModel.get(dbm, form_id)
xform = questionnaire.xform
if xform:
xform_cleaned = re.sub(
r"\s+", " ",
re.sub(r"\n", "",
questionnaire.xform_with_unique_ids_substituted()))
questionnaire.name = escape(questionnaire.name)
#so that in the smartphone repeat questions have atleast one group pre added
xform_cleaned = re.sub(
r"<html:title>.*</html:title>",
r"<html:title>" + questionnaire.name + "</html:title>",
xform_cleaned)
xform_cleaned = re.sub(
r"<html:title />",
r"<html:title>" + questionnaire.name + "</html:title>",
xform_cleaned)
return re.sub('ns2:template=""', "", xform_cleaned)
_escape_special_characters(questionnaire)
ui_fields = []
for field in questionnaire.fields:
if isinstance(field, UniqueIdField):
ui_fields.append(UniqueIdUIField(field, dbm))
else:
ui_fields.append(field)
template = env.get_template('reporter_entity_form.xml')
return template.render(questionnaire=questionnaire,
fields=ui_fields,
field_xmls=field_xmls,
reporter_id=reporter_id,
field_types=field_types,
default_template=env.get_template('text_field.xml'))
def get_submission_breakup(request, project_id):
dbm = get_database_manager(request.user)
project = Project.load(dbm.database, project_id)
form_model = FormModel.get(dbm, project.qid)
submission_success, submission_errors = submission_stats(dbm, form_model.form_code)
response = json.dumps([submission_success, submission_errors])
return HttpResponse(response)
def save_questionnaire(request):
manager = get_database_manager(request.user)
if request.method == 'POST':
questionnaire_code = request.POST['questionnaire-code']
json_string = request.POST['question-set']
question_set = json.loads(json_string)
pid = request.POST['pid']
project = Project.load(manager.database, pid)
form_model = FormModel.get(manager, project.qid)
old_fields = form_model.fields
try:
QuestionnaireBuilder(form_model, manager).update_questionnaire_with_questions(question_set)
except QuestionCodeAlreadyExistsException as e:
return HttpResponseServerError(e)
except QuestionAlreadyExistsException as e:
return HttpResponseServerError(e)
except EntityQuestionAlreadyExistsException as e:
return HttpResponseServerError(e.message)
else:
try:
form_model.form_code = questionnaire_code.lower()
except DataObjectAlreadyExists as e:
if e.message.find("Form") >= 0:
return HttpResponseServerError("Questionnaire with this code already exists")
return HttpResponseServerError(e.message)
form_model.name = project.name
form_model.entity_id = project.entity_type
detail = utils.get_changed_questions(old_fields, form_model.fields, subject=False)
form_model.save()
UserActivityLog().log(request, project=project.name, action=EDITED_PROJECT, detail=json.dumps(detail))
return HttpResponse(json.dumps({"response": "ok"}))
def test_should_add_integer_field_with_constraints(self):
integer_question = FormModel.get(self.manager,
self.form_model_id).fields[2]
range_constraint = integer_question.constraints[0]
self.assertTrue(integer_question.name == "Father's age")
self.assertTrue(range_constraint.min, 15)
self.assertTrue(range_constraint.max, 120)
def build(self):
if not entity_type_already_defined(self._manager, self._entity_type):
define_type(self._manager, self._entity_type)
self.form_model = FormModel(self._manager, name=self._name, label=self._label,
form_code=self._form_code, fields=self._fields, is_registration_model=self._is_reg)
form_model_id = self.form_model.save()
return FormModel.get(self._manager, form_model_id)
def test_should_raise_exception_if_label_is_not_unique(self):
with self.assertRaises(QuestionAlreadyExistsException):
form_model = FormModel.get(self.manager, self.form_model_id)
question = TextField(name="added_question",
code="q5",
label="What is your name")
form_model.add_field(question)
form_model.save()
def edit_project(request, project_id=None):
manager = get_database_manager(request.user)
entity_list = get_all_entity_types(manager)
entity_list = helper.remove_reporter(entity_list)
project = Project.load(manager.database, project_id)
questionnaire = FormModel.get(manager, project.qid)
if request.method == 'GET':
form = CreateProject(data=project, entity_list=entity_list)
activity_report_questions = json.dumps(helper.get_activity_report_questions(manager), default=field_to_json)
subject_report_questions = json.dumps(helper.get_subject_report_questions(manager), default=field_to_json)
fields = questionnaire.fields
if questionnaire.entity_defaults_to_reporter():
fields = helper.hide_entity_question(questionnaire.fields)
existing_questions = json.dumps(fields, default=field_to_json)
return render_to_response('project/create_project.html',
{'form': form, "activity_report_questions": repr(activity_report_questions),
'subject_report_questions': repr(subject_report_questions),
'preview_links': get_preview_and_instruction_links(),
'existing_questions': repr(existing_questions), 'questionnaire_code': questionnaire.form_code,
'project': project, 'is_edit': 'true', 'post_url': reverse(edit_project, args=[project_id])},
context_instance=RequestContext(request))
if request.method == 'POST':
project_info = json.loads(request.POST['profile_form'])
form = CreateProject(entity_list, data=project_info)
if form.is_valid():
detail = dict()
for key, changed in enumerate(form.changed_data):
if getattr(project, changed) != form.cleaned_data.get(changed):
detail.update({changed.capitalize(): form.cleaned_data.get(changed)})
project.update(form.cleaned_data)
try:
old_fields = questionnaire.fields
questionnaire = update_questionnaire(questionnaire, request.POST, form.cleaned_data['entity_type'],
form.cleaned_data['name'], manager, form.cleaned_data['language'])
changed_questions = get_changed_questions(old_fields, questionnaire.fields, subject=False)
detail.update(changed_questions)
project.state = request.POST['project_state']
project.qid = questionnaire.save()
UserActivityLog().log(request, project=project.name, action=EDITED_PROJECT, detail=json.dumps(detail))
except (QuestionCodeAlreadyExistsException, QuestionAlreadyExistsException,
EntityQuestionAlreadyExistsException) as ex:
return HttpResponse(
json.dumps({'success': False, 'error_in_project_section': False, 'error_message': _(ex.message)}))
except DataObjectAlreadyExists:
return HttpResponse(json.dumps({'success': False, 'error_in_project_section': False,
'error_message': 'Questionnaire with this code already exists'}))
try:
project.save(manager)
except DataObjectAlreadyExists as ex:
message = _("%s with %s = %s already exists.") % (_(ex.data[2]), _(ex.data[0]), "'%s'" % project.name)
return HttpResponse(
json.dumps({'success': False, 'error_message': message, 'error_in_project_section': True}))
return HttpResponse(json.dumps({'success': True, 'project_id': project.id}))
def broadcast_message(request, project_id):
dbm = get_database_manager(request.user)
project = Project.load(dbm.database, project_id)
number_associated_ds = len(project.data_senders)
number_of_ds = len(import_module.load_all_entities_of_type(dbm, type=REPORTER)[0]) - 1
questionnaire = FormModel.get(dbm, project.qid)
organization = utils.get_organization(request)
if request.method == 'GET':
form = BroadcastMessageForm(associated_ds=number_associated_ds, number_of_ds=number_of_ds)
html = 'project/broadcast_message_trial.html' if organization.in_trial_mode else 'project/broadcast_message.html'
return render_to_response(html, {'project': project,
"project_links": make_project_links(project, questionnaire.form_code),
'is_quota_reached': is_quota_reached(request, organization=organization),
"form": form, "ong_country": organization.country,
"success": None},
context_instance=RequestContext(request))
if request.method == 'POST':
form = BroadcastMessageForm(associated_ds=number_associated_ds, number_of_ds=number_of_ds, data=request.POST)
if form.is_valid():
no_smsc = False
data_senders = _get_data_senders(dbm, form, project)
organization_setting = OrganizationSetting.objects.get(organization=organization)
current_month = datetime.date(datetime.datetime.now().year, datetime.datetime.now().month, 1)
message_tracker = organization._get_message_tracker(current_month)
other_numbers = form.cleaned_data['others']
failed_numbers = []
try:
failed_numbers = helper.broadcast_message(data_senders, form.cleaned_data['text'],
organization_setting.get_organisation_sms_number()[0],
other_numbers,
message_tracker,
country_code=organization.get_phone_country_code())
except NoSMSCException as e:
no_smsc = True
success = not no_smsc and len(failed_numbers) == 0
if success:
form = BroadcastMessageForm(associated_ds=number_associated_ds, number_of_ds=number_of_ds)
else:
form = BroadcastMessageForm(associated_ds=number_associated_ds, number_of_ds=number_of_ds,
data=request.POST)
return render_to_response('project/broadcast_message.html',
{'project': project,
"project_links": make_project_links(project, questionnaire.form_code),
'is_quota_reached': is_quota_reached(request, organization=organization),
"form": form,
"ong_country": organization.country, "no_smsc": no_smsc,
'failed_numbers': ",".join(failed_numbers), "success": success},
context_instance=RequestContext(request))
return render_to_response('project/broadcast_message.html',
{'project': project,
"project_links": make_project_links(project, questionnaire.form_code), "form": form,
'is_quota_reached': is_quota_reached(request, organization=organization),
'success': None, "ong_country": organization.country},
context_instance=RequestContext(request))
def get_report_data(dbm, config, keys, index):
questionnaire = FormModel.get(dbm, config.questionnaires[0]["id"])
enrichable_questions = questionnaire.special_questions()
_load_entities_to_entity_questions(dbm, enrichable_questions)
rows = get_survey_response_by_report_view_name(
dbm, "report_" + config.id + "_" + index, keys)
return [{
config.questionnaires[0]["alias"]:
_enrich_questions(dbm, row, questionnaire, enrichable_questions)
} for row in rows]
def __init__(self, request, project_id=None):
self.request = request
self.manager = get_database_manager(self.request.user)
self.project = Project.load(self.manager.database, project_id)
self.form_model = FormModel.get(self.manager, self.project.qid)
self.form_code = self.form_model.form_code
self.feeds_dbm = get_feeds_database(request.user)
self.subject_field_creator = SubjectQuestionFieldCreator(self.manager, self.project)
self.is_data_sender = self.request.user.get_profile().reporter
self.disable_link_class, self.hide_link_class = get_visibility_settings_for(self.request.user)
def update_submission_search_index(submission_doc, dbm, refresh_index=True):
es = get_elasticsearch_handle()
form_model = FormModel.get(dbm, submission_doc.form_model_id)
search_dict = _meta_fields(submission_doc, dbm)
_update_with_form_model_fields(dbm, submission_doc, search_dict,
form_model)
es.index(dbm.database_name,
form_model.id,
search_dict,
id=submission_doc.id,
refresh=refresh_index)
def test_should_add_new_field(self):
form_model = FormModel.get(self.manager, self.form_model_id)
question = TextField(name="added_question",
code="Q4",
label="How are you")
form_model.add_field(question)
form_model.save()
added_question = self.manager.get(self.form_model.id,
FormModel).fields[4]
self.assertEquals(added_question.code, "Q4")
def async_reindex(db_name, form_model_id):
response = dict()
response['db_name'] = db_name
response['questionnaire_id'] = form_model_id
response['start_time'] = time.time()
dbm = get_db_manager(db_name)
form_model = FormModel.get(dbm, form_model_id)
submission_search_store = SubmissionSearchStore(dbm, form_model, old_form_model=None)
submission_search_store.recreate_elastic_store()
_populate_submission_index(db_name, form_model_id)
response['end_time'] = time.time()
return response
def test_should_get_latest_field_if_no_revision_provided_and_no_snapshots(
self):
self.form_model.delete_field(code="Q3")
field = SelectField(name="New Name",
code="Q3",
label="What is your favourite color",
options=[("RED", 1), ("YELLOW", 2)])
self.form_model.add_field(field)
self.form_model.save()
updated_form = FormModel.get(self.manager, self.form_model_id)
self.assertEqual("New Name",
updated_form.get_field_by_code_and_rev("Q3").name)
def delete_survey(self, survey_response, additional_details):
feed_delete_errors = None
try:
survey_response.void()
form_model = FormModel.get(self.dbm, survey_response.form_model_id)
if self.feeds_dbm:
feed_delete_errors = EnrichedSurveyResponseBuilder(
self.dbm, survey_response, form_model,
additional_details).delete_feed_document(self.feeds_dbm)
except MangroveException as e:
return Response(errors=e.message,
feed_error_message=feed_delete_errors)
return Response(success=True, feed_error_message=feed_delete_errors)
def sent_reminders(request, project_id):
dbm = get_database_manager(request.user)
project = Project.load(dbm.database, project_id)
questionnaire = FormModel.get(dbm, project.qid)
organization = Organization.objects.get(org_id=request.user.get_profile().org_id)
is_trial_account = organization.in_trial_mode
html = 'project/reminders_trial.html' if organization.in_trial_mode else 'project/sent_reminders.html'
return render_to_response(html,
{'project': project,
"project_links": make_project_links(project, questionnaire.form_code),
'is_quota_reached': is_quota_reached(request, organization=organization),
'reminders': get_all_reminder_logs_for_project(project_id, dbm),
'in_trial_mode': is_trial_account,
'create_reminder_link': reverse(create_reminder, args=[project_id])},
context_instance=RequestContext(request))
def get_submission_breakup(request, project_id):
dbm = get_database_manager(request.user)
project = Project.load(dbm.database, project_id)
form_model = FormModel.get(dbm, project.qid)
rows = dbm.load_all_rows_in_view('undeleted_submission_log',
startkey=[form_model.form_code],
endkey=[form_model.form_code, {}],
group=True,
group_level=1,
reduce=True)
submission_success, submission_errors = 0, 0
for row in rows:
submission_success = row["value"]["success"]
submission_errors = row["value"]["count"] - row["value"]["success"]
response = json.dumps([submission_success, submission_errors])
return HttpResponse(response)
def get_submissions_about_project(request, project_id):
dbm = get_database_manager(request.user)
project = Project.load(dbm.database, project_id)
form_model = FormModel.get(dbm, project.qid)
rows = dbm.load_all_rows_in_view('undeleted_survey_response', reduce=False, descending=True, startkey=[form_model.form_code, {}],
endkey=[form_model.form_code], limit=7)
submission_list = []
for row in rows:
reporter = _find_reporter_name(dbm, row)
message = _make_message(row)
submission = dict(message=message, created=row.value["submitted_on"].strftime("%B %d %y %H:%M"), reporter=reporter,
status=row.value["status"])
submission_list.append(submission)
submission_response = json.dumps(submission_list)
return HttpResponse(submission_response)
def activate_project(request, project_id=None):
manager = get_database_manager(request.user)
project = Project.load(manager.database, project_id)
project.activate(manager)
form_model = FormModel.get(manager, project.qid)
oneDay = datetime.timedelta(days=1)
tomorrow = datetime.datetime.now() + oneDay
survey_responses = get_survey_responses(manager, form_model.form_code, from_time=0,
to_time=int(mktime(tomorrow.timetuple())) * 1000, page_size=None)
feeds_dbm = get_feeds_database(request.user)
service = SurveyResponseService(manager, logger, feeds_dbm)
additional_feed_dictionary = get_project_details_dict_for_feed(project)
for survey_response in survey_responses:
service.delete_survey(survey_response, additional_feed_dictionary)
UserActivityLog().log(request, action=ACTIVATED_PROJECT, project=project.name)
return HttpResponseRedirect(reverse('project-overview', args=[project_id]))
def check_for_name_mismatch(dbm):
for row in dbm.database.query(list_all_projects, include_docs=True):
try:
project_data = row.doc
form_model = FormModel.get(dbm, project_data.get("qid"))
form_model_doc = form_model._doc
if form_model_doc['name'] != project_data['name']:
logging.debug(
"name mismatch for database %s, project with_id %s" %
(dbm.database_name, row.id))
except Exception as e:
logging.error(
'something failed for for database : %s, project_doc with id: %s'
% (dbm.database_name, row.id))
logging.error(e)
def recreate_subject_index(db_name):
logger = logging.getLogger(db_name)
try:
logger.info('Starting indexing')
dbm = get_db_manager(db_name)
form_models = dbm.database.query(map_form_model_for_subjects)
es = get_elasticsearch_handle()
for row in form_models:
try:
form_model = FormModel.get(dbm, row.id)
entity_type = form_model.entity_type[0]
try:
es.delete_all(db_name, entity_type)
except Exception as ignore:
pass
create_subject_mapping(dbm, form_model)
entity_docs = []
for entity_doc in get_all_entities_include_voided(
dbm, [entity_type]):
try:
if entity_doc.data:
subject = subject_dict(entity_type, entity_doc,
dbm, form_model)
subject.update({'id': entity_doc.id})
entity_docs.append(subject)
except Exception as e:
logger.error("Failed to index subject with id %s" %
entity_doc.id)
logger.error(e)
if entity_docs:
es.bulk_index(dbm.database_name, entity_type, entity_docs)
es.refresh(dbm.database_name)
logger.info('Changed index for subject with codes ' +
str([a.get('id') for a in entity_docs]))
except Exception as e:
logger.error("Failed to create subject mapping for %s" %
row.id)
logger.error(e.message)
logger.info('Completed Indexing')
mark_as_completed(db_name)
except Exception as e:
logger.exception(e.message)
