def update_submission_search_for_subject_edition(entity_doc, dbm):
from datawinners.search.submission_query import SubmissionQueryBuilder
entity_type = entity_doc.entity_type
projects = []
for row in dbm.load_all_rows_in_view('projects_by_subject_type',
key=entity_type[0],
include_docs=True):
projects.append(
Project.new_from_doc(dbm, ProjectDocument.wrap(row['doc'])))
for project in projects:
entity_field_code = None
for field in project.entity_questions:
if [field.unique_id_type] == entity_type:
entity_field_code = field.code
if entity_field_code:
unique_id_field_name = es_field_name(entity_field_code, project.id)
fields_mapping = {
unique_id_field_name: entity_doc.data['name']['value']
}
args = {
es_unique_id_code_field_name(unique_id_field_name):
entity_doc.short_code
}
survey_response_filtered_query = SubmissionQueryBuilder(
project).query_all(dbm.database_name, project.id, **args)
for survey_response in survey_response_filtered_query.all():
SubmissionIndexUpdateHandler(
dbm.database_name,
project.id).update_field_in_submission_index(
survey_response._id, fields_mapping)
def get_headers(self, user, subject_type):
manager = get_database_manager(user)
form_model = get_form_model_by_entity_type(manager, [subject_type])
#return header_fields(form_model,key_attribute='code').keys()
subject_es_headers = []
for code in header_fields(form_model, key_attribute='code').keys():
subject_es_headers.append(es_field_name(code, form_model.id))
return subject_es_headers
def get_mappings(self):
fields_definition = []
fields_definition.extend(get_submission_meta_fields())
for field in self.form_model.fields:
fields_definition.append(
get_field_definition(field, field_name=es_field_name(field.code, self.form_model.id)))
mapping = self.get_fields_mapping_by_field_def(doc_type=self.form_model.id, fields_definition=fields_definition)
return mapping
def test_submission_status_headers_for_errored_submissions(self):
self.form_model.fields = [self.field1, self.field2, self.field3, self.field4]
self.form_model.entity_questions = [self.field3, self.field4]
headers = ErroredSubmissionHeader(self.form_model).get_header_field_names()
expected = [es_field_name(f, self.form_model.id) for f in
["ds_id", "ds_name", "date","error_msg", "q1", "q2", "q3", "q3_unique_code", "q4", "q4_unique_code"]]
self.assertListEqual(expected, headers)
def test_should_return_submission_log_specific_header_fields(self):
self.form_model.fields = [self.field1, self.field2, self.field3, self.field4]
self.form_model.entity_questions = [self.field3, self.field4]
headers = AllSubmissionHeader(self.form_model).get_header_field_names()
expected = [es_field_name(f, self.form_model.id) for f in
["ds_id", "ds_name", "date", "status", "q1", "q2", "q3", "q3_unique_code", "q4", "q4_unique_code"]]
self.assertListEqual(expected, headers)
def update_static_header_info(self):
header_dict = OrderedDict()
header_dict.update(
{SubmissionIndexConstants.DATASENDER_ID_KEY: "Datasender Id"})
header_dict.update(
{SubmissionIndexConstants.DATASENDER_NAME_KEY: "Datasender Name"})
header_dict.update({"date": "Submission Date"})
subject_title = self.form_model.entity_type[0].title()
header_dict.update({
es_field_name(self.form_model.entity_question.code, self.form_model.id):
subject_title
})
header_dict.update({'entity_short_code': "%s ID" % subject_title})
if self.form_model.event_time_question:
header_dict.update({
es_field_name(self.form_model.event_time_question.code, self.form_model.id):
"Reporting Date"
})
return header_dict
def get_header_dict(self):
header_dict = OrderedDict()
header_dict.update(self.update_static_header_info())
def key_attribute(field):
return field.code.lower()
headers = header_fields(self.form_model, key_attribute)
for field_code, val in headers.items():
key = es_field_name(field_code, self.form_model.id)
if not header_dict.has_key(key): header_dict.update({key: val})
if self.form_model.is_entity_type_reporter():
# For summary projects there will be an extra datasender info question(code eid).
# This condition removes that extra question.
header_dict.pop(
es_field_name(self.form_model.entity_question.code,
self.form_model.id))
header_dict.pop('entity_short_code')
return header_dict
def create_subject_mapping(dbm, form_model):
es = get_elasticsearch_handle()
fields_definition = []
for field in form_model.fields:
fields_definition.append(
get_field_definition(field,
field_name=es_field_name(
field.code, form_model.id)))
mapping = get_fields_mapping_by_field_def(
doc_type=form_model.id, fields_definition=fields_definition)
es.put_mapping(dbm.database_name, form_model.entity_type[0], mapping)
def get_mappings(self):
fields_definition = []
fields_definition.extend(get_submission_meta_fields())
for field in self.latest_form_model.fields:
if isinstance(field, UniqueIdField):
unique_id_field_name = es_field_name(field.code,
self.latest_form_model.id)
fields_definition.append(
get_field_definition(
field,
field_name=es_unique_id_code_field_name(
unique_id_field_name)))
fields_definition.append(
get_field_definition(field,
field_name=es_field_name(
field.code,
self.latest_form_model.id)))
mapping = self.get_fields_mapping_by_field_def(
doc_type=self.latest_form_model.id,
fields_definition=fields_definition)
return mapping
def get_column_title(self):
header = HeaderFactory(self._form_model).create_header(
self.submission_type)
header_dict = header.get_header_field_dict()
header_dict.pop('ds_id', None)
unique_question_field_names = [
es_unique_id_code_field_name(
es_field_name(field.code, self._form_model.id))
for field in self._form_model.entity_questions
]
for field_name in unique_question_field_names:
header_dict.pop(field_name, None)
return header_dict.values()
def test_submission_status_headers_for_success_and_erred_submissions(self):
form_model = MagicMock(spec=FormModel, id="2323")
form_model.is_entity_type_reporter.return_value = False
entity_question_field = Mock(spec=Field)
form_model.entity_question = entity_question_field
form_model.event_time_question = None
entity_question_field.code.lower.return_value = 'eid'
with patch("datawinners.search.submission_headers.header_fields") as header_fields:
header_fields.return_value = {}
query_params = {"filter": "success"}
headers = SubmissionQuery(form_model, query_params).get_headers(Mock, "code")
expected = [es_field_name(f, "2323") for f in ["ds_id", "ds_name", "date", "eid", "entity_short_code"]]
self.assertListEqual(expected, headers)
query_params = {"filter": "error"}
headers = SubmissionQuery(form_model, query_params).get_headers(Mock, "code")
expected = [es_field_name(f, "2323") for f in
["ds_id", "ds_name", "date", "error_msg", "eid", "entity_short_code"]]
self.assertListEqual(expected, headers)
def subject_autocomplete(request, entity_type):
search_text = lower(request.GET["term"] or "")
database_name = get_database_name(request.user)
dbm = get_database_manager(request.user)
form_model = get_form_model_by_entity_type(dbm, [entity_type.lower()])
subject_name_field = get_field_by_attribute_value(form_model, 'name',
'name')
es_field_name_for_subject_name = es_field_name(subject_name_field.code,
form_model.id)
subject_short_code_field = get_field_by_attribute_value(
form_model, 'name', 'short_code')
es_field_name_for_short_code = es_field_name(subject_short_code_field.code,
form_model.id)
query = elasticutils.S().es(urls=ELASTIC_SEARCH_URL).indexes(database_name).doctypes(lower(entity_type)) \
.query(or_={es_field_name_for_subject_name + '__match': search_text,
es_field_name_for_subject_name + '_value': search_text,
es_field_name_for_short_code + '__match': search_text,
es_field_name_for_short_code + '_value': search_text}) \
.values_dict()
resp = [{
"id": r[es_field_name_for_short_code],
"label": r[es_field_name_for_subject_name]
} for r in query[:min(query.count(), 50)]]
return HttpResponse(json.dumps(resp))
def _add_unique_id_filters(self, query, uniqueIdFilters):
if uniqueIdFilters:
for uniqueIdType, uniqueIdFilter in uniqueIdFilters.iteritems():
if uniqueIdFilter:
search_options = elasticutils.F()
for question in [
question
for question in self.form_model.entity_questions
if question.unique_id_type == uniqueIdType
]:
es_field_code = es_unique_id_code_field_name(
es_field_name(question.code, self.form_model.id))
search_options |= elasticutils.F(
**{es_field_code: uniqueIdFilter})
query = query.filter(search_options)
return query
def test_should_return_submission_log_specific_header_fields(self):
form_model = MagicMock(spec=FormModel, id="2323")
#form_model.entity_type = ["clinic"]
entity_question_field = Mock(spec=Field)
form_model.entity_question = entity_question_field
form_model.event_time_question = None
form_model.is_entity_type_reporter.return_value = False
entity_question_field.code.lower.return_value = 'eid'
with patch("datawinners.search.submission_headers.header_fields") as header_fields:
header_fields.return_value = {}
query_params = {"filter": "all"}
headers = SubmissionQuery(form_model, query_params).get_headers()
expected = [es_field_name(f, "2323") for f in
["ds_id", "ds_name", "date", "status", "eid", "entity_short_code"]]
self.assertListEqual(expected, headers)
def update_submission_search_for_subject_edition(entity_doc, dbm):
from datawinners.search.submission_query import SubmissionQueryBuilder
entity_type = entity_doc.entity_type
projects = dbm.load_all_rows_in_view('projects_by_subject_type', key=entity_type[0], include_docs=True)
form_models = _get_form_models_from_projects(dbm, projects)
for form_model in form_models:
entity_field_name = lower(form_model.entity_question.code)
fields_mapping = {es_field_name(entity_field_name, form_model.id): entity_doc.data['name']['value']}
args = {'entity_short_code_value': entity_doc.short_code}
survey_response_filtered_query = SubmissionQueryBuilder(form_model).query_all(dbm.database_name, form_model.id,
**args)
for survey_response in survey_response_filtered_query.all():
SubmissionIndexUpdateHandler(dbm.database_name, form_model.id).update_field_in_submission_index(
survey_response._id, fields_mapping)
def create_response(self, required_field_names, query):
entity_question_codes = [
es_field_name(field.code, self.form_model.id)
for field in self.form_model.entity_questions
]
meta_fields = [SubmissionIndexConstants.DATASENDER_ID_KEY]
meta_fields.extend([
es_unique_id_code_field_name(code)
for code in entity_question_codes
])
submissions = []
language = get_language()
for res in query.values_dict(tuple(required_field_names)):
submission = [res._id]
for key in required_field_names:
if not key in meta_fields:
if key in entity_question_codes:
self.combine_name_and_id(short_code=res.get(
es_unique_id_code_field_name(key)),
entity_name=res.get(key),
submission=submission)
elif key == SubmissionIndexConstants.DATASENDER_NAME_KEY:
self.combine_name_and_id(
res.get(
SubmissionIndexConstants.DATASENDER_ID_KEY),
res.get(
SubmissionIndexConstants.DATASENDER_NAME_KEY),
submission)
elif key == 'status':
submission.append(ugettext(res.get(key)))
elif key == 'error_msg':
error_msg = res.get(key)
if error_msg.find('| |') != -1:
error_msg = error_msg.split('| |,')[[
'en', 'fr'
].index(language)]
submission.append(error_msg)
else:
submission.append(res.get(key))
submissions.append(submission)
return submissions
def test_should_have_reporting_date_header_if_form_model_has_reporting_date(self):
form_model = MagicMock(spec=FormModel, id="2323")
form_model.is_entity_type_reporter.return_value = False
entity_question_field = Mock(spec=Field)
form_model.entity_question = entity_question_field
entity_question_field.code.lower.return_value = 'eid'
event_time_field = Mock(spec=Field)
form_model.event_time_question = event_time_field
event_time_field.code.lower.return_value = "rp_date"
with patch("datawinners.search.submission_headers.header_fields") as header_fields:
header_fields.return_value = {}
query_params = {"filter": "success"}
headers = SubmissionQuery(form_model, query_params).get_headers(Mock, "code")
expected = [es_field_name(f, "2323") for f in
["ds_id", "ds_name", "date", "eid", "entity_short_code", "rp_date"]]
self.assertListEqual(expected, headers)
def get_facet_response_for_choice_fields(query_with_criteria, choice_fields, form_model_id):
facet_results = []
for field in choice_fields:
field_name = es_field_name(field.code, form_model_id) + "_exact"
facet_response = query_with_criteria.facet(field_name, filtered=True).facet_counts()
facet_result_options = []
facet_result = {
"es_field_name": field_name,
"facets": facet_result_options,
# find total submissions containing specified answer
"total": query_with_criteria.filter(~F(**{field_name: None})).count()
}
for option, facet_list in facet_response.iteritems():
for facet in facet_list:
facet_result_options.append({
"term": facet['term'],
"count": facet['count']
})
facet_results.append(facet_result)
return facet_results
def _update_with_form_model_fields(dbm, submission_doc, search_dict, form_model):
#Submission value may have capitalized keys in some cases. This conversion is to do
#case insensitive lookup.
submission_values = OrderedDict((k.lower(), v) for k,v in submission_doc.values.iteritems())
for field in form_model.fields:
entry = submission_values.get(lower(field.code))
if field.is_entity_field:
entity_name = lookup_entity_name(dbm, entry, form_model.entity_type)
entry_code = UNKNOWN if entity_name == UNKNOWN else entry
search_dict.update({"entity_short_code": entry_code or UNKNOWN})
entry = entity_name
elif field.type == "select":
field = _update_select_field_by_revision(field, form_model, submission_doc)
if field.type == "select":
entry = field.get_option_value_list(entry)
elif field.type == "select1":
entry = ",".join(field.get_option_value_list(entry))
elif field.type == "select1":
field = _update_select_field_by_revision(field, form_model, submission_doc)
if field.type == "select":
entry = field.get_option_value_list(entry)
elif field.type == "select1":
entry = ",".join(field.get_option_value_list(entry))
elif field.type == "date":
try:
if form_model.revision != submission_doc.form_model_revision:
old_submission_value = entry
to_format = field.date_format
current_format = form_model.get_field_by_code_and_rev(field.code, submission_doc.form_model_revision).__date__(entry)
entry = current_format.strftime(DateField.DATE_DICTIONARY.get(to_format))
logger.info("Converting old date submission from %s to %s" % (old_submission_value, entry))
except Exception as ignore_conversion_errors:
pass
if entry:
search_dict.update({es_field_name(lower(field.code), form_model.id): entry})
search_dict.update({'void': submission_doc.void})
return search_dict
def get_columns(self):
header = HeaderFactory(self._form_model).create_header(
self.submission_type)
header_dict = header.get_header_field_dict()
# header_dict = OrderedDict({'name':"Name"}, {"p":"Place", "})
if self._form_model.entity_type == ["reporter"]:
header_dict.pop("entity_short_code", None)
result = OrderedDict()
for key in header_dict:
if key != SubmissionIndexConstants.DATASENDER_ID_KEY:
result.update({key: {"label": header_dict[key]}})
if key == SubmissionIndexConstants.DATASENDER_NAME_KEY: #add key column after name
self.add_datasender_id_column(header_dict, result)
for field in self._form_model.fields:
field_name = es_field_name(field.code.lower(), self._form_model.id)
if result.has_key(field_name):
result.get(field_name).update({"type": field.type})
if field.type == "date":
result.get(field_name).update(
{"format": field.date_format})
return result
def _update_with_form_model_fields(dbm, submission_doc, search_dict,
form_model):
#Submission value may have capitalized keys in some cases. This conversion is to do
#case insensitive lookup.
submission_values = OrderedDict(
(k.lower(), v) for k, v in submission_doc.values.iteritems())
for field in form_model.fields:
entry = submission_values.get(lower(field.code))
if field.is_entity_field:
if entry:
original_field = form_model.get_field_by_code_and_rev(
field.code, submission_doc.form_model_revision)
if is_original_question_changed_from_choice_answer_type(
original_field, field):
entry = convert_choice_options_to_options_text(
original_field, entry)
entity_name = lookup_entity_name(dbm, entry,
[field.unique_id_type])
entry_code = entry
search_dict.update({
es_unique_id_code_field_name(
es_field_name(lower(field.code), form_model.id)):
entry_code or UNKNOWN
})
entry = entity_name
elif field.type == "select":
field = _update_select_field_by_revision(field, form_model,
submission_doc)
if field.type == "select":
entry = field.get_option_value_list(entry)
elif field.type == "select1":
entry = ",".join(field.get_option_value_list(entry))
elif field.type == "select1":
field = _update_select_field_by_revision(field, form_model,
submission_doc)
if field.type == "select":
entry = field.get_option_value_list(entry)
elif field.type == "select1":
entry = ",".join(field.get_option_value_list(entry))
elif field.type == 'text':
field_for_revision = form_model.get_field_by_code_and_rev(
field.code, submission_doc.form_model_revision)
if isinstance(field_for_revision, SelectField):
entry = _get_select_field_answer_from_snapshot(
entry, field_for_revision)
elif field.type == "date":
try:
if form_model.revision != submission_doc.form_model_revision:
old_submission_value = entry
to_format = field.date_format
field_for_revision = form_model.get_field_by_code_and_rev(
field.code, submission_doc.form_model_revision)
if isinstance(field_for_revision, DateField):
current_date = field_for_revision.__date__(entry)
entry = current_date.strftime(
DateField.DATE_DICTIONARY.get(to_format))
elif isinstance(field_for_revision, SelectField):
entry = _get_select_field_answer_from_snapshot(
entry, field_for_revision)
logger.info(
"Converting old date submission from %s to %s" %
(old_submission_value, entry))
except Exception as ignore_conversion_errors:
pass
if entry:
search_dict.update(
{es_field_name(lower(field.code), form_model.id): entry})
search_dict.update({'void': submission_doc.void})
return search_dict
def get_date_field_name(self):
return es_field_name(self.reporting_date_field.code,
self.form_model.id)
def get_date_field_name(self):
return es_field_name(ES_SUBMISSION_FIELD_DATE, None)
