def _get_submission_fields(self,
fields_definition,
fields,
parent_field_name=None):
for field in fields:
if isinstance(field, UniqueIdField):
unique_id_field_name = es_questionnaire_field_name(
field.code, self.latest_form_model.id, parent_field_name)
fields_definition.append(
get_field_definition(
field,
field_name=es_unique_id_code_field_name(
unique_id_field_name)))
if isinstance(field, FieldSet):
if field.is_group():
self._get_submission_fields(fields_definition,
field.fields, field.code)
else:
es_field_name = es_questionnaire_field_name(
field.code, self.latest_form_model.id,
parent_field_name)
fields_definition.append(
get_field_definition_with_binary_type(
field, field_name=es_field_name))
continue
fields_definition.append(
get_field_definition(field,
field_name=es_questionnaire_field_name(
field.code, self.latest_form_model.id,
parent_field_name)))
def create_response(self, required_field_names, search_results, search_parameters):
entity_question_codes = [es_questionnaire_field_name(field.code, self.form_model.id, field.parent_field_code)
for field in
self.form_model.entity_questions]
fieldset_fields = get_field_set_fields(self.form_model.id, self.form_model.fields)
meta_fields = [SubmissionIndexConstants.DATASENDER_ID_KEY]
meta_fields.extend([es_unique_id_code_field_name(code) for code in entity_question_codes])
media_field_codes = self._get_media_field_codes()
image_fields = self._get_image_field_codes()
submissions = []
language = get_language()
if hasattr(search_results, 'aggregations'):
aggr_result = []
groups = [0]
self._traverse_aggregation_buckets(search_results.aggregations, aggr_result, groups)
if search_parameters.get('search_filters').get('duplicatesForFilter') == 'exactmatch':
aggr_result = self._group_and_filter_aggregation(aggr_result, groups)
for res in aggr_result:
submission = [res._id]
group_id = res.group_id
res = res._source
self._append_to_submission(entity_question_codes, fieldset_fields, image_fields, language,
media_field_codes, meta_fields, required_field_names, res, submission)
submission.append(group_id)
submissions.append(submission)
return submissions, len(aggr_result)
else:
for res in search_results.hits:
submission = [res.meta.id]
self._append_to_submission(entity_question_codes, fieldset_fields, image_fields, language,
media_field_codes, meta_fields, required_field_names, res, submission)
submission.append(0)
submissions.append(submission)
return submissions, search_results.hits.total
def get_header_dict(self):
header_dict = OrderedDict()
header_dict.update(self.update_static_header_info())
def key_attribute(field):
return field.code
if isinstance(self.form_model, EntityFormModel):
entity_questions = self.form_model.base_entity_questions
else:
entity_questions = self.form_model.entity_questions
entity_question_dict = dict(
(self._get_entity_question_path(field), field)
for field in entity_questions)
headers = header_fields(self.form_model, key_attribute)
for field_code, val in headers.items():
if field_code in entity_question_dict.keys():
self.add_unique_id_field(entity_question_dict.get(field_code),
header_dict)
else:
key = es_questionnaire_field_name(field_code.replace(".", " "),
self.form_model.id)
header_dict.update({key: val})
return header_dict
def get_facet_response_for_choice_fields(query_with_criteria, choice_fields,
form_model_id):
facet_results = []
for field in choice_fields:
field_name = es_questionnaire_field_name(field.code,
form_model_id) + "_exact"
facet_response = query_with_criteria.facet(
field_name, filtered=True).facet_counts()
facet_result_options = []
facet_result = {
"es_field_name": field_name,
"facets": facet_result_options,
# find total submissions containing specified answer
"total":
query_with_criteria.filter(~F(**{field_name: None})).count()
}
for option, facet_list in facet_response.iteritems():
for facet in facet_list:
facet_result_options.append({
"term": facet['term'],
"count": facet['count']
})
facet_results.append(facet_result)
return facet_results
def _update_with_field_meta(self,
fields,
result,
header,
parent_field_name=None):
for field in fields:
if isinstance(field, FieldSet) and field.is_group():
self._update_with_field_meta(field.fields, result, header,
field.code)
else:
field_name = es_questionnaire_field_name(
field.code, self._form_model.id, parent_field_name)
if field_name in result.keys():
result.get(field_name).update({
"type": field.type,
"code": field.code
})
if field.type == "date":
result.get(field_name).update(
{"format": field.date_format})
if field.type == "field_set":
result.get(field_name).update({
"fields":
self.get_sub_fields_of(field, header),
"code":
field.code,
'fieldset_type':
field.fieldset_type
})
def get_facets_for_choice_fields(dbm, form_model, search_parameters,
local_time_delta):
query_fields, search = _create_query(dbm, form_model, local_time_delta,
search_parameters)
query_body = search.to_dict()
es = Elasticsearch(hosts=[{
"host": ELASTIC_SEARCH_HOST,
"port": ELASTIC_SEARCH_PORT
}])
total_submission_count = get_submission_count(dbm, form_model,
search_parameters,
local_time_delta)
facet_results = []
for field in form_model.choice_fields:
field_name = es_questionnaire_field_name(field.code,
form_model.id) + "_exact"
facet = _create_facet_request_body(field_name, query_body)
facet_response = es.search(index=dbm.database_name,
doc_type=form_model.id,
body=facet,
search_type='count')
facet_result = _get_facet_result(facet_response, field_name)
facet_results.append(facet_result)
return facet_results, total_submission_count
def update_submission_search_for_subject_edition(dbm, unique_id_type,
short_code, last_name):
projects = []
for row in dbm.load_all_rows_in_view('projects_by_subject_type',
key=unique_id_type[0],
include_docs=True):
projects.append(
Project.new_from_doc(dbm, ProjectDocument.wrap(row['doc'])))
for project in projects:
entity_field_code = None
for field in project.entity_questions:
if [field.unique_id_type] == unique_id_type:
entity_field_code = field.code
if entity_field_code:
unique_id_field_name = es_questionnaire_field_name(
entity_field_code, project.id)
fields_mapping = {unique_id_field_name: last_name}
args = {
es_unique_id_code_field_name(unique_id_field_name): short_code
}
query = _get_submissions_for_unique_id_entry(args, dbm, project)
for survey_response in query.values_dict('void'):
SubmissionIndexUpdateHandler(
dbm.database_name,
project.id).update_field_in_submission_index(
survey_response._id, fields_mapping)
def create_response(self, required_field_names, search_results):
entity_question_codes = [es_questionnaire_field_name(field.code, self.form_model.id, field.parent_field_code) for field in
self.form_model.entity_questions]
fieldset_fields = self.get_field_set_fields(self.form_model.fields)
meta_fields = [SubmissionIndexConstants.DATASENDER_ID_KEY]
meta_fields.extend([es_unique_id_code_field_name(code) for code in entity_question_codes])
media_field_codes = self._get_media_field_codes()
image_fields = self._get_image_field_codes()
submissions = []
language = get_language()
for res in search_results.hits:
submission = [res._meta.id]
for key in required_field_names:
if not key in meta_fields:
if key in entity_question_codes:
self.combine_name_and_id(short_code=res.get(es_unique_id_code_field_name(key)),
entity_name=res.get(key), submission=submission)
elif key == SubmissionIndexConstants.DATASENDER_NAME_KEY:
self._populate_datasender(res, submission)
elif key == 'status':
submission.append(ugettext(res.get(key)))
elif key == SubmissionIndexConstants.SUBMISSION_DATE_KEY:
self._convert_to_localized_date_time(key, res, submission)
elif key == 'error_msg':
self._populate_error_message(key, language, res, submission)
elif key in fieldset_fields.keys():
submission.append(
_format_fieldset_values_for_representation(res.get(key), fieldset_fields.get(key),
res._meta.id))
else:
submission.append(self._append_if_attachments_are_present(res, key, media_field_codes, image_fields))
submissions.append(submission)
return submissions
def _get_image_field_codes(self):
media_field_code = []
for media_field in self.form_model.media_fields:
if isinstance(media_field, PhotoField):
media_field_code.append(
es_questionnaire_field_name(media_field.code, self.form_model.id, media_field.parent_field_code))
return media_field_code
def get_headers(self, user, subject_type):
manager = get_database_manager(user)
form_model = get_form_model_by_entity_type(manager, [subject_type])
subject_es_headers = []
for code in header_fields(form_model, key_attribute='code').keys():
subject_es_headers.append(
es_questionnaire_field_name(code, form_model.id))
return subject_es_headers
def _get_field_name(field, questionnaire_id):
parent_code = field.get('parent_field_code') if field.get(
'parent_field_code') else None
field_name = es_questionnaire_field_name(field['code'], questionnaire_id,
parent_code)
field_suffix = '_value' if field['type'] == 'date' \
else '_unique_code' if field['type'] == 'unique_id_exact' \
else '_exact'
return field_name + field_suffix
def _get_unique_id_es_field_name(field,
project_id,
entity_questionnaire=False):
unique_id_field_name = es_questionnaire_field_name(field.code, project_id)
if not entity_questionnaire:
unique_id_field_name += '_unique_code_exact'
else:
unique_id_field_name += "_unique_code"
return unique_id_field_name
def subject_autocomplete(request, entity_type):
search_text = lower(request.GET["term"] or "")
database_name = get_database_name(request.user)
dbm = get_database_manager(request.user)
form_model = get_form_model_by_entity_type(dbm, [entity_type.lower()])
subject_name_field = get_field_by_attribute_value(form_model, 'name', 'name')
es_field_name_for_subject_name = es_questionnaire_field_name(subject_name_field.code, form_model.id)
subject_short_code_field = get_field_by_attribute_value(form_model, 'name', 'short_code')
es_field_name_for_short_code = es_questionnaire_field_name(subject_short_code_field.code, form_model.id)
query = elasticutils.S().es(urls=ELASTIC_SEARCH_URL, timeout=ELASTIC_SEARCH_TIMEOUT).indexes(database_name).doctypes(lower(entity_type)) \
.query(or_={es_field_name_for_subject_name + '__match': search_text,
es_field_name_for_subject_name + '_value': search_text,
es_field_name_for_short_code + '__match': search_text,
es_field_name_for_short_code + '_value': search_text}) \
.values_dict()
resp = [{"id": r[es_field_name_for_short_code], "label": r[es_field_name_for_subject_name]} for r in
query[:min(query.count(), 50)]]
return HttpResponse(json.dumps(resp))
def get_field_set_fields(self, fields, parent_field_code=None):
field_set_field_dict = {}
for field in fields:
if isinstance(field, FieldSet):
field_set_field_dict.update(
{es_questionnaire_field_name(field.code, self.form_model.id, parent_field_code): field})
group_field_code = field.code if field.is_group() else None
field_set_field_dict.update(self.get_field_set_fields(field.fields, group_field_code))
return field_set_field_dict
def get_column_title(self):
header = HeaderFactory(self._form_model).create_header(self.submission_type)
header_dict = header.get_header_field_dict()
header_dict.pop('ds_id', None)
unique_question_field_names = [es_unique_id_code_field_name(es_questionnaire_field_name(field.code, self._form_model.id, field.parent_field_code)) for
field in
self._form_model.entity_questions]
for field_name in unique_question_field_names:
header_dict.pop(field_name, None)
return header_dict.values()
def create_subject_mapping(dbm, form_model):
es = get_elasticsearch_handle()
fields_definition = []
for field in form_model.fields:
fields_definition.append(
get_field_definition(field,
field_name=es_questionnaire_field_name(
field.code, form_model.id)))
mapping = get_fields_mapping_by_field_def(
doc_type=form_model.id, fields_definition=fields_definition)
es.put_mapping(dbm.database_name, form_model.entity_type[0], mapping)
def get_field_names_as_header_name(self):
headers = self.get_header_dict().keys()
entity_questions = self.form_model.entity_questions
for entity_question in entity_questions:
if not self.form_model.is_part_of_repeat_field(entity_question):
headers.remove(
es_unique_id_code_field_name(
es_questionnaire_field_name(
entity_question.code, self.form_model.id,
entity_question.parent_field_code)))
return headers
def add_unique_id_field(self, unique_id_field, header_dict):
unique_id_question_code = unique_id_field.code
subject_title = unique_id_field.unique_id_type
unique_id_field_name = es_questionnaire_field_name(
unique_id_question_code, self.form_model.id,
unique_id_field.parent_field_code)
header_dict.update({unique_id_field_name: unique_id_field.label})
header_dict.update({
es_unique_id_code_field_name(unique_id_field_name):
"%s ID" % subject_title
})
def get_aggregations_for_choice_fields(dbm, form_model, local_time_delta,
pagination_params, sort_params,
search_parameters):
search = _create_search(dbm, form_model, local_time_delta,
pagination_params, sort_params, search_parameters)
search = search.params(search_type="count")
field_names = []
for field in form_model.choice_fields:
field_name = es_questionnaire_field_name(field.code, form_model.id)
a = A("terms", field=field_name + '_exact', size=0)
search.aggs.bucket(field_name, a)
field_names.append(field_name)
search_results = search.execute()
aggs_results = [
_get_aggregation_result(field_name, search_results)
for field_name in field_names
]
return aggs_results, search_results.hits.total
def _add_unique_id_filters(form_model, unique_id_filters, search):
if unique_id_filters:
for uniqueIdType, uniqueIdFilter in unique_id_filters.iteritems():
if uniqueIdFilter:
unique_id_filters = []
for question in [
question for question in form_model.entity_questions
if question.unique_id_type == uniqueIdType
]:
es_field_code = es_unique_id_code_field_name(
es_questionnaire_field_name(
question.code,
form_model.id,
parent_field_code=question.parent_field_code)
) + "_exact"
unique_id_filters.append(
F("term", **{es_field_code: uniqueIdFilter}))
search = search.filter(F('or', unique_id_filters))
return search
def _update_header_dict_from_fields(self,
fields,
header_dict,
parent_field_code=None):
for field in fields:
if field.is_entity_field:
self.add_unique_id_in_header_dict(header_dict,
field,
parent_field_code,
parent_field_types=[],
nested=False)
elif isinstance(field, FieldSet) and field.is_group():
self._update_header_dict_from_fields(field.fields, header_dict,
field.code)
else:
key = es_questionnaire_field_name(
field.code.replace(".", " "),
self.form_model.id,
parent_field_code=parent_field_code)
header_dict.update({key: field.label})
def _get_unique_id_es_field_name(field, project_id):
unique_id_field_name = es_questionnaire_field_name(field.code, project_id)
return unique_id_field_name + '_unique_code_exact'
def _update_search_dict(dbm,
form_model,
fields,
search_dict,
submission_doc,
submission_values,
parent_field_name=None):
for field in fields:
entry = submission_values.get(field.code)
label_to_be_displayed = get_label_to_be_displayed(
entry, field, form_model, submission_doc)
es_field_name = es_questionnaire_field_name(field.code, form_model.id,
parent_field_name)
if label_to_be_displayed:
if 'media' not in search_dict.keys():
search_dict.update({'media': {}})
if isinstance(field, PhotoField):
search_dict['media'].update({
es_field_name: {
'type':
'image',
'value':
entry,
'download_link':
'/download/attachment/' + submission_doc.id + '/' +
entry,
'preview_link':
'/download/attachment/' + submission_doc.id +
'/preview_' + entry,
}
})
if isinstance(field, AudioField):
search_dict['media'].update({
es_field_name: {
'type':
'audio',
'value':
entry,
'download_link':
'/download/attachment/' + submission_doc.id + '/' +
entry,
}
})
if isinstance(field, VideoField):
search_dict['media'].update({
es_field_name: {
'type':
'video',
'value':
entry,
'download_link':
'/download/attachment/' + submission_doc.id + '/' +
entry,
}
})
if isinstance(field, FieldSet):
if field.is_group():
for value in entry:
_update_search_dict(dbm, form_model, field.fields,
search_dict, submission_doc, value,
field.code)
else:
_update_repeat_fields_with_choice_values(
label_to_be_displayed, field)
_update_name_unique_code(dbm, label_to_be_displayed, field)
search_dict.update(
{es_field_name: json.dumps(label_to_be_displayed)})
elif field.is_entity_field:
entity, choice_to_entity_entry = get_entity(
dbm, entry, field, form_model, submission_doc)
if entry:
search_dict.update({
es_unique_id_details_field_name(es_field_name):
entity
})
search_dict.update({
es_unique_id_code_field_name(es_field_name):
choice_to_entity_entry or UNKNOWN
})
search_dict.update({es_field_name: entry and entity.get('q2')})
else:
search_dict.update({es_field_name: label_to_be_displayed})
search_dict.update({'void': submission_doc.void})
search_dict.update(
{'is_anonymous': submission_doc.is_anonymous_submission})
def _get_media_field_codes(self):
return [es_questionnaire_field_name(field.code, self.form_model.id, field.parent_field_code) for
field in
self.form_model.media_fields] if self.form_model.is_media_type_fields_present else []
def _update_search_dict(dbm, form_model, fields, search_dict, submission_doc, submission_values,
parent_field_name=None):
for field in fields:
field_code = field.code
entry = submission_values.get(field_code)
if field.is_entity_field:
if entry:
original_field = form_model.get_field_by_code_and_rev(field.code, submission_doc.form_model_revision)
if is_original_question_changed_from_choice_answer_type(original_field, field):
entry = convert_choice_options_to_options_text(original_field, entry)
entity_name = lookup_entity_name(dbm, entry, [field.unique_id_type])
entry_code = entry
search_dict.update(
{es_unique_id_code_field_name(
es_questionnaire_field_name(field.code, form_model.id, parent_field_name)): entry_code or UNKNOWN})
entry = entity_name
elif field.type == "select":
field = _get_select_field_by_revision(field, form_model, submission_doc)
if field.type == "select":
entry = field.get_option_value_list(entry)
elif field.type == "select1":
entry = ",".join(field.get_option_value_list(entry))
elif field.type == "select1":
field = _get_select_field_by_revision(field, form_model, submission_doc)
if field.type == "select":
entry = field.get_option_value_list(entry)
elif field.type == "select1":
entry = ",".join(field.get_option_value_list(entry))
elif field.type == 'text':
field_for_revision = form_model.get_field_by_code_and_rev(field.code, submission_doc.form_model_revision)
if isinstance(field_for_revision, SelectField):
entry = _get_select_field_answer_from_snapshot(entry, field_for_revision)
elif field.type == "date":
try:
if form_model.revision != submission_doc.form_model_revision:
old_submission_value = entry
to_format = field.date_format
field_for_revision = form_model.get_field_by_code_and_rev(field.code,
submission_doc.form_model_revision)
if isinstance(field_for_revision, DateField):
current_date = field_for_revision.__date__(entry)
entry = current_date.strftime(DateField.DATE_DICTIONARY.get(to_format))
elif isinstance(field_for_revision, SelectField):
entry = _get_select_field_answer_from_snapshot(entry, field_for_revision)
logger.info("Converting old date submission from %s to %s" % (old_submission_value, entry))
except Exception as ignore_conversion_errors:
pass
if entry:
if isinstance(field, FieldSet):
if field.is_group():
for value in submission_values[field_code]:
_update_search_dict(dbm, form_model, field.fields, search_dict, submission_doc, value,
field.code)
else:
_update_repeat_fields_with_choice_values(entry, field)
_update_name_unique_code(dbm, entry, field)
search_dict.update(
{es_questionnaire_field_name(field_code, form_model.id, parent_field_name): json.dumps(entry)})
else:
search_dict.update({es_questionnaire_field_name(field.code, form_model.id, parent_field_name): entry})
search_dict.update({'void': submission_doc.void})
search_dict.update({'is_anonymous': submission_doc.is_anonymous_submission})
def _field_code(self, field):
return es_questionnaire_field_name(field['code'], self.form_model.id,
field.get('parent_field_code'))
def get_date_field_name(self):
# Assumption is that date filters won't appear for dates questions within repeat fields
return es_questionnaire_field_name(
self.date_field.code, self.form_model.id,
self.date_field.parent_field_code) + "_value"
