def create_response(self, required_field_names, search_results):
entity_question_codes = [es_questionnaire_field_name(field.code, self.form_model.id, field.parent_field_code) for field in
self.form_model.entity_questions]
fieldset_fields = self.get_field_set_fields(self.form_model.fields)
meta_fields = [SubmissionIndexConstants.DATASENDER_ID_KEY]
meta_fields.extend([es_unique_id_code_field_name(code) for code in entity_question_codes])
media_field_codes = self._get_media_field_codes()
image_fields = self._get_image_field_codes()
submissions = []
language = get_language()
for res in search_results.hits:
submission = [res._meta.id]
for key in required_field_names:
if not key in meta_fields:
if key in entity_question_codes:
self.combine_name_and_id(short_code=res.get(es_unique_id_code_field_name(key)),
entity_name=res.get(key), submission=submission)
elif key == SubmissionIndexConstants.DATASENDER_NAME_KEY:
self._populate_datasender(res, submission)
elif key == 'status':
submission.append(ugettext(res.get(key)))
elif key == SubmissionIndexConstants.SUBMISSION_DATE_KEY:
self._convert_to_localized_date_time(key, res, submission)
elif key == 'error_msg':
self._populate_error_message(key, language, res, submission)
elif key in fieldset_fields.keys():
submission.append(
_format_fieldset_values_for_representation(res.get(key), fieldset_fields.get(key),
res._meta.id))
else:
submission.append(self._append_if_attachments_are_present(res, key, media_field_codes, image_fields))
submissions.append(submission)
return submissions
def update_submission_search_for_subject_edition(entity_doc, dbm):
from datawinners.search.submission_query import SubmissionQueryBuilder
entity_type = entity_doc.entity_type
projects = []
for row in dbm.load_all_rows_in_view('projects_by_subject_type',
key=entity_type[0],
include_docs=True):
projects.append(
Project.new_from_doc(dbm, ProjectDocument.wrap(row['doc'])))
for project in projects:
entity_field_code = None
for field in project.entity_questions:
if [field.unique_id_type] == entity_type:
entity_field_code = field.code
if entity_field_code:
unique_id_field_name = es_field_name(entity_field_code, project.id)
fields_mapping = {
unique_id_field_name: entity_doc.data['name']['value']
}
args = {
es_unique_id_code_field_name(unique_id_field_name):
entity_doc.short_code
}
survey_response_filtered_query = SubmissionQueryBuilder(
project).query_all(dbm.database_name, project.id, **args)
for survey_response in survey_response_filtered_query.all():
SubmissionIndexUpdateHandler(
dbm.database_name,
project.id).update_field_in_submission_index(
survey_response._id, fields_mapping)
def _append_to_submission(self, entity_question_codes, fieldset_fields,
image_fields, language, media_field_codes,
meta_fields, required_field_names, res,
submission):
for key in required_field_names:
if key not in meta_fields:
if key in entity_question_codes:
self.combine_name_and_id(short_code=safe_getattr(
res, es_unique_id_code_field_name(key)),
entity_name=safe_getattr(
res, key),
submission=submission)
elif key == SubmissionIndexConstants.DATASENDER_NAME_KEY:
self._populate_datasender(res, submission)
elif key == 'status':
submission.append(ugettext(safe_getattr(res, key)))
elif key == SubmissionIndexConstants.SUBMISSION_DATE_KEY:
self._convert_to_localized_date_time(key, res, submission)
elif key == 'error_msg':
self._populate_error_message(key, language, res,
submission)
elif key in fieldset_fields.keys():
submission.append(
format_fieldset_values_for_representation(
safe_getattr(res, key), fieldset_fields.get(key),
submission[0]))
else:
submission.append(
self._append_if_attachments_are_present(
res, key, media_field_codes, image_fields,
submission[0]))
def update_submission_search_for_subject_edition(dbm, unique_id_type,
short_code, last_name):
projects = []
for row in dbm.load_all_rows_in_view('projects_by_subject_type',
key=unique_id_type[0],
include_docs=True):
projects.append(
Project.new_from_doc(dbm, ProjectDocument.wrap(row['doc'])))
for project in projects:
entity_field_code = None
for field in project.entity_questions:
if [field.unique_id_type] == unique_id_type:
entity_field_code = field.code
if entity_field_code:
unique_id_field_name = es_questionnaire_field_name(
entity_field_code, project.id)
fields_mapping = {unique_id_field_name: last_name}
args = {
es_unique_id_code_field_name(unique_id_field_name): short_code
}
query = _get_submissions_for_unique_id_entry(args, dbm, project)
for survey_response in query.values_dict('void'):
SubmissionIndexUpdateHandler(
dbm.database_name,
project.id).update_field_in_submission_index(
survey_response._id, fields_mapping)
def create_response(self, required_field_names, search_results, search_parameters):
entity_question_codes = [es_questionnaire_field_name(field.code, self.form_model.id, field.parent_field_code)
for field in
self.form_model.entity_questions]
fieldset_fields = get_field_set_fields(self.form_model.id, self.form_model.fields)
meta_fields = [SubmissionIndexConstants.DATASENDER_ID_KEY]
meta_fields.extend([es_unique_id_code_field_name(code) for code in entity_question_codes])
media_field_codes = self._get_media_field_codes()
image_fields = self._get_image_field_codes()
submissions = []
language = get_language()
if hasattr(search_results, 'aggregations'):
aggr_result = []
groups = [0]
self._traverse_aggregation_buckets(search_results.aggregations, aggr_result, groups)
if search_parameters.get('search_filters').get('duplicatesForFilter') == 'exactmatch':
aggr_result = self._group_and_filter_aggregation(aggr_result, groups)
for res in aggr_result:
submission = [res._id]
group_id = res.group_id
res = res._source
self._append_to_submission(entity_question_codes, fieldset_fields, image_fields, language,
media_field_codes, meta_fields, required_field_names, res, submission)
submission.append(group_id)
submissions.append(submission)
return submissions, len(aggr_result)
else:
for res in search_results.hits:
submission = [res.meta.id]
self._append_to_submission(entity_question_codes, fieldset_fields, image_fields, language,
media_field_codes, meta_fields, required_field_names, res, submission)
submission.append(0)
submissions.append(submission)
return submissions, search_results.hits.total
def _get_submission_fields(self,
fields_definition,
fields,
parent_field_name=None):
for field in fields:
if isinstance(field, UniqueIdField):
unique_id_field_name = es_questionnaire_field_name(
field.code, self.latest_form_model.id, parent_field_name)
fields_definition.append(
get_field_definition(
field,
field_name=es_unique_id_code_field_name(
unique_id_field_name)))
if isinstance(field, FieldSet):
if field.is_group():
self._get_submission_fields(fields_definition,
field.fields, field.code)
else:
es_field_name = es_questionnaire_field_name(
field.code, self.latest_form_model.id,
parent_field_name)
fields_definition.append(
get_field_definition_with_binary_type(
field, field_name=es_field_name))
continue
fields_definition.append(
get_field_definition(field,
field_name=es_questionnaire_field_name(
field.code, self.latest_form_model.id,
parent_field_name)))
def create_response(self, required_field_names, query):
entity_question_codes = [
es_field_name(field.code, self.form_model.id)
for field in self.form_model.entity_questions
]
meta_fields = [SubmissionIndexConstants.DATASENDER_ID_KEY]
meta_fields.extend([
es_unique_id_code_field_name(code)
for code in entity_question_codes
])
submissions = []
language = get_language()
for res in query.values_dict(tuple(required_field_names)):
submission = [res._id]
for key in required_field_names:
if not key in meta_fields:
if key in entity_question_codes:
self.combine_name_and_id(short_code=res.get(
es_unique_id_code_field_name(key)),
entity_name=res.get(key),
submission=submission)
elif key == SubmissionIndexConstants.DATASENDER_NAME_KEY:
self.combine_name_and_id(
res.get(
SubmissionIndexConstants.DATASENDER_ID_KEY),
res.get(
SubmissionIndexConstants.DATASENDER_NAME_KEY),
submission)
elif key == 'status':
submission.append(ugettext(res.get(key)))
elif key == 'error_msg':
error_msg = res.get(key)
if error_msg.find('| |') != -1:
error_msg = error_msg.split('| |,')[[
'en', 'fr'
].index(language)]
submission.append(error_msg)
else:
submission.append(res.get(key))
submissions.append(submission)
return submissions
def get_column_title(self):
header = HeaderFactory(self._form_model).create_header(self.submission_type)
header_dict = header.get_header_field_dict()
header_dict.pop('ds_id', None)
unique_question_field_names = [es_unique_id_code_field_name(es_questionnaire_field_name(field.code, self._form_model.id, field.parent_field_code)) for
field in
self._form_model.entity_questions]
for field_name in unique_question_field_names:
header_dict.pop(field_name, None)
return header_dict.values()
def get_field_names_as_header_name(self):
headers = self.get_header_dict().keys()
entity_questions = self.form_model.entity_questions
for entity_question in entity_questions:
if not self.form_model.is_part_of_repeat_field(entity_question):
headers.remove(
es_unique_id_code_field_name(
es_questionnaire_field_name(
entity_question.code, self.form_model.id,
entity_question.parent_field_code)))
return headers
def add_unique_id_field(self, unique_id_field, header_dict):
unique_id_question_code = unique_id_field.code
subject_title = unique_id_field.unique_id_type
unique_id_field_name = es_questionnaire_field_name(
unique_id_question_code, self.form_model.id,
unique_id_field.parent_field_code)
header_dict.update({unique_id_field_name: unique_id_field.label})
header_dict.update({
es_unique_id_code_field_name(unique_id_field_name):
"%s ID" % subject_title
})
def _add_unique_id_filters(self, query, uniqueIdFilters):
if uniqueIdFilters:
for uniqueIdType, uniqueIdFilter in uniqueIdFilters.iteritems():
if uniqueIdFilter:
search_options = elasticutils.F()
for question in [
question
for question in self.form_model.entity_questions
if question.unique_id_type == uniqueIdType
]:
es_field_code = es_unique_id_code_field_name(
es_field_name(question.code, self.form_model.id))
search_options |= elasticutils.F(
**{es_field_code: uniqueIdFilter})
query = query.filter(search_options)
return query
def _add_unique_id_filters(form_model, unique_id_filters, search):
if unique_id_filters:
for uniqueIdType, uniqueIdFilter in unique_id_filters.iteritems():
if uniqueIdFilter:
unique_id_filters = []
for question in [
question for question in form_model.entity_questions
if question.unique_id_type == uniqueIdType
]:
es_field_code = es_unique_id_code_field_name(
es_questionnaire_field_name(
question.code,
form_model.id,
parent_field_code=question.parent_field_code)
) + "_exact"
unique_id_filters.append(
F("term", **{es_field_code: uniqueIdFilter}))
search = search.filter(F('or', unique_id_filters))
return search
def get_mappings(self):
fields_definition = []
fields_definition.extend(get_submission_meta_fields())
for field in self.latest_form_model.fields:
if isinstance(field, UniqueIdField):
unique_id_field_name = es_field_name(field.code,
self.latest_form_model.id)
fields_definition.append(
get_field_definition(
field,
field_name=es_unique_id_code_field_name(
unique_id_field_name)))
fields_definition.append(
get_field_definition(field,
field_name=es_field_name(
field.code,
self.latest_form_model.id)))
mapping = self.get_fields_mapping_by_field_def(
doc_type=self.latest_form_model.id,
fields_definition=fields_definition)
return mapping
def _update_with_form_model_fields(dbm, submission_doc, search_dict,
form_model):
#Submission value may have capitalized keys in some cases. This conversion is to do
#case insensitive lookup.
submission_values = OrderedDict(
(k.lower(), v) for k, v in submission_doc.values.iteritems())
for field in form_model.fields:
entry = submission_values.get(lower(field.code))
if field.is_entity_field:
if entry:
original_field = form_model.get_field_by_code_and_rev(
field.code, submission_doc.form_model_revision)
if is_original_question_changed_from_choice_answer_type(
original_field, field):
entry = convert_choice_options_to_options_text(
original_field, entry)
entity_name = lookup_entity_name(dbm, entry,
[field.unique_id_type])
entry_code = entry
search_dict.update({
es_unique_id_code_field_name(
es_field_name(lower(field.code), form_model.id)):
entry_code or UNKNOWN
})
entry = entity_name
elif field.type == "select":
field = _update_select_field_by_revision(field, form_model,
submission_doc)
if field.type == "select":
entry = field.get_option_value_list(entry)
elif field.type == "select1":
entry = ",".join(field.get_option_value_list(entry))
elif field.type == "select1":
field = _update_select_field_by_revision(field, form_model,
submission_doc)
if field.type == "select":
entry = field.get_option_value_list(entry)
elif field.type == "select1":
entry = ",".join(field.get_option_value_list(entry))
elif field.type == 'text':
field_for_revision = form_model.get_field_by_code_and_rev(
field.code, submission_doc.form_model_revision)
if isinstance(field_for_revision, SelectField):
entry = _get_select_field_answer_from_snapshot(
entry, field_for_revision)
elif field.type == "date":
try:
if form_model.revision != submission_doc.form_model_revision:
old_submission_value = entry
to_format = field.date_format
field_for_revision = form_model.get_field_by_code_and_rev(
field.code, submission_doc.form_model_revision)
if isinstance(field_for_revision, DateField):
current_date = field_for_revision.__date__(entry)
entry = current_date.strftime(
DateField.DATE_DICTIONARY.get(to_format))
elif isinstance(field_for_revision, SelectField):
entry = _get_select_field_answer_from_snapshot(
entry, field_for_revision)
logger.info(
"Converting old date submission from %s to %s" %
(old_submission_value, entry))
except Exception as ignore_conversion_errors:
pass
if entry:
search_dict.update(
{es_field_name(lower(field.code), form_model.id): entry})
search_dict.update({'void': submission_doc.void})
return search_dict
def _update_search_dict(dbm, form_model, fields, search_dict, submission_doc, submission_values,
parent_field_name=None):
for field in fields:
field_code = field.code
entry = submission_values.get(field_code)
if field.is_entity_field:
if entry:
original_field = form_model.get_field_by_code_and_rev(field.code, submission_doc.form_model_revision)
if is_original_question_changed_from_choice_answer_type(original_field, field):
entry = convert_choice_options_to_options_text(original_field, entry)
entity_name = lookup_entity_name(dbm, entry, [field.unique_id_type])
entry_code = entry
search_dict.update(
{es_unique_id_code_field_name(
es_questionnaire_field_name(field.code, form_model.id, parent_field_name)): entry_code or UNKNOWN})
entry = entity_name
elif field.type == "select":
field = _get_select_field_by_revision(field, form_model, submission_doc)
if field.type == "select":
entry = field.get_option_value_list(entry)
elif field.type == "select1":
entry = ",".join(field.get_option_value_list(entry))
elif field.type == "select1":
field = _get_select_field_by_revision(field, form_model, submission_doc)
if field.type == "select":
entry = field.get_option_value_list(entry)
elif field.type == "select1":
entry = ",".join(field.get_option_value_list(entry))
elif field.type == 'text':
field_for_revision = form_model.get_field_by_code_and_rev(field.code, submission_doc.form_model_revision)
if isinstance(field_for_revision, SelectField):
entry = _get_select_field_answer_from_snapshot(entry, field_for_revision)
elif field.type == "date":
try:
if form_model.revision != submission_doc.form_model_revision:
old_submission_value = entry
to_format = field.date_format
field_for_revision = form_model.get_field_by_code_and_rev(field.code,
submission_doc.form_model_revision)
if isinstance(field_for_revision, DateField):
current_date = field_for_revision.__date__(entry)
entry = current_date.strftime(DateField.DATE_DICTIONARY.get(to_format))
elif isinstance(field_for_revision, SelectField):
entry = _get_select_field_answer_from_snapshot(entry, field_for_revision)
logger.info("Converting old date submission from %s to %s" % (old_submission_value, entry))
except Exception as ignore_conversion_errors:
pass
if entry:
if isinstance(field, FieldSet):
if field.is_group():
for value in submission_values[field_code]:
_update_search_dict(dbm, form_model, field.fields, search_dict, submission_doc, value,
field.code)
else:
_update_repeat_fields_with_choice_values(entry, field)
_update_name_unique_code(dbm, entry, field)
search_dict.update(
{es_questionnaire_field_name(field_code, form_model.id, parent_field_name): json.dumps(entry)})
else:
search_dict.update({es_questionnaire_field_name(field.code, form_model.id, parent_field_name): entry})
search_dict.update({'void': submission_doc.void})
search_dict.update({'is_anonymous': submission_doc.is_anonymous_submission})
def _update_search_dict(dbm,
form_model,
fields,
search_dict,
submission_doc,
submission_values,
parent_field_name=None):
for field in fields:
entry = submission_values.get(field.code)
label_to_be_displayed = get_label_to_be_displayed(
entry, field, form_model, submission_doc)
es_field_name = es_questionnaire_field_name(field.code, form_model.id,
parent_field_name)
if label_to_be_displayed:
if 'media' not in search_dict.keys():
search_dict.update({'media': {}})
if isinstance(field, PhotoField):
search_dict['media'].update({
es_field_name: {
'type':
'image',
'value':
entry,
'download_link':
'/download/attachment/' + submission_doc.id + '/' +
entry,
'preview_link':
'/download/attachment/' + submission_doc.id +
'/preview_' + entry,
}
})
if isinstance(field, AudioField):
search_dict['media'].update({
es_field_name: {
'type':
'audio',
'value':
entry,
'download_link':
'/download/attachment/' + submission_doc.id + '/' +
entry,
}
})
if isinstance(field, VideoField):
search_dict['media'].update({
es_field_name: {
'type':
'video',
'value':
entry,
'download_link':
'/download/attachment/' + submission_doc.id + '/' +
entry,
}
})
if isinstance(field, FieldSet):
if field.is_group():
for value in entry:
_update_search_dict(dbm, form_model, field.fields,
search_dict, submission_doc, value,
field.code)
else:
_update_repeat_fields_with_choice_values(
label_to_be_displayed, field)
_update_name_unique_code(dbm, label_to_be_displayed, field)
search_dict.update(
{es_field_name: json.dumps(label_to_be_displayed)})
elif field.is_entity_field:
entity, choice_to_entity_entry = get_entity(
dbm, entry, field, form_model, submission_doc)
if entry:
search_dict.update({
es_unique_id_details_field_name(es_field_name):
entity
})
search_dict.update({
es_unique_id_code_field_name(es_field_name):
choice_to_entity_entry or UNKNOWN
})
search_dict.update({es_field_name: entry and entity.get('q2')})
else:
search_dict.update({es_field_name: label_to_be_displayed})
search_dict.update({'void': submission_doc.void})
search_dict.update(
{'is_anonymous': submission_doc.is_anonymous_submission})
