def create_response(self, required_field_names, search_results): entity_question_codes = [es_questionnaire_field_name(field.code, self.form_model.id, field.parent_field_code) for field in self.form_model.entity_questions] fieldset_fields = self.get_field_set_fields(self.form_model.fields) meta_fields = [SubmissionIndexConstants.DATASENDER_ID_KEY] meta_fields.extend([es_unique_id_code_field_name(code) for code in entity_question_codes]) media_field_codes = self._get_media_field_codes() image_fields = self._get_image_field_codes() submissions = [] language = get_language() for res in search_results.hits: submission = [res._meta.id] for key in required_field_names: if not key in meta_fields: if key in entity_question_codes: self.combine_name_and_id(short_code=res.get(es_unique_id_code_field_name(key)), entity_name=res.get(key), submission=submission) elif key == SubmissionIndexConstants.DATASENDER_NAME_KEY: self._populate_datasender(res, submission) elif key == 'status': submission.append(ugettext(res.get(key))) elif key == SubmissionIndexConstants.SUBMISSION_DATE_KEY: self._convert_to_localized_date_time(key, res, submission) elif key == 'error_msg': self._populate_error_message(key, language, res, submission) elif key in fieldset_fields.keys(): submission.append( _format_fieldset_values_for_representation(res.get(key), fieldset_fields.get(key), res._meta.id)) else: submission.append(self._append_if_attachments_are_present(res, key, media_field_codes, image_fields)) submissions.append(submission) return submissions
def update_submission_search_for_subject_edition(entity_doc, dbm): from datawinners.search.submission_query import SubmissionQueryBuilder entity_type = entity_doc.entity_type projects = [] for row in dbm.load_all_rows_in_view('projects_by_subject_type', key=entity_type[0], include_docs=True): projects.append( Project.new_from_doc(dbm, ProjectDocument.wrap(row['doc']))) for project in projects: entity_field_code = None for field in project.entity_questions: if [field.unique_id_type] == entity_type: entity_field_code = field.code if entity_field_code: unique_id_field_name = es_field_name(entity_field_code, project.id) fields_mapping = { unique_id_field_name: entity_doc.data['name']['value'] } args = { es_unique_id_code_field_name(unique_id_field_name): entity_doc.short_code } survey_response_filtered_query = SubmissionQueryBuilder( project).query_all(dbm.database_name, project.id, **args) for survey_response in survey_response_filtered_query.all(): SubmissionIndexUpdateHandler( dbm.database_name, project.id).update_field_in_submission_index( survey_response._id, fields_mapping)
def _append_to_submission(self, entity_question_codes, fieldset_fields, image_fields, language, media_field_codes, meta_fields, required_field_names, res, submission): for key in required_field_names: if key not in meta_fields: if key in entity_question_codes: self.combine_name_and_id(short_code=safe_getattr( res, es_unique_id_code_field_name(key)), entity_name=safe_getattr( res, key), submission=submission) elif key == SubmissionIndexConstants.DATASENDER_NAME_KEY: self._populate_datasender(res, submission) elif key == 'status': submission.append(ugettext(safe_getattr(res, key))) elif key == SubmissionIndexConstants.SUBMISSION_DATE_KEY: self._convert_to_localized_date_time(key, res, submission) elif key == 'error_msg': self._populate_error_message(key, language, res, submission) elif key in fieldset_fields.keys(): submission.append( format_fieldset_values_for_representation( safe_getattr(res, key), fieldset_fields.get(key), submission[0])) else: submission.append( self._append_if_attachments_are_present( res, key, media_field_codes, image_fields, submission[0]))
def update_submission_search_for_subject_edition(dbm, unique_id_type, short_code, last_name): projects = [] for row in dbm.load_all_rows_in_view('projects_by_subject_type', key=unique_id_type[0], include_docs=True): projects.append( Project.new_from_doc(dbm, ProjectDocument.wrap(row['doc']))) for project in projects: entity_field_code = None for field in project.entity_questions: if [field.unique_id_type] == unique_id_type: entity_field_code = field.code if entity_field_code: unique_id_field_name = es_questionnaire_field_name( entity_field_code, project.id) fields_mapping = {unique_id_field_name: last_name} args = { es_unique_id_code_field_name(unique_id_field_name): short_code } query = _get_submissions_for_unique_id_entry(args, dbm, project) for survey_response in query.values_dict('void'): SubmissionIndexUpdateHandler( dbm.database_name, project.id).update_field_in_submission_index( survey_response._id, fields_mapping)
def create_response(self, required_field_names, search_results, search_parameters): entity_question_codes = [es_questionnaire_field_name(field.code, self.form_model.id, field.parent_field_code) for field in self.form_model.entity_questions] fieldset_fields = get_field_set_fields(self.form_model.id, self.form_model.fields) meta_fields = [SubmissionIndexConstants.DATASENDER_ID_KEY] meta_fields.extend([es_unique_id_code_field_name(code) for code in entity_question_codes]) media_field_codes = self._get_media_field_codes() image_fields = self._get_image_field_codes() submissions = [] language = get_language() if hasattr(search_results, 'aggregations'): aggr_result = [] groups = [0] self._traverse_aggregation_buckets(search_results.aggregations, aggr_result, groups) if search_parameters.get('search_filters').get('duplicatesForFilter') == 'exactmatch': aggr_result = self._group_and_filter_aggregation(aggr_result, groups) for res in aggr_result: submission = [res._id] group_id = res.group_id res = res._source self._append_to_submission(entity_question_codes, fieldset_fields, image_fields, language, media_field_codes, meta_fields, required_field_names, res, submission) submission.append(group_id) submissions.append(submission) return submissions, len(aggr_result) else: for res in search_results.hits: submission = [res.meta.id] self._append_to_submission(entity_question_codes, fieldset_fields, image_fields, language, media_field_codes, meta_fields, required_field_names, res, submission) submission.append(0) submissions.append(submission) return submissions, search_results.hits.total
def _get_submission_fields(self, fields_definition, fields, parent_field_name=None): for field in fields: if isinstance(field, UniqueIdField): unique_id_field_name = es_questionnaire_field_name( field.code, self.latest_form_model.id, parent_field_name) fields_definition.append( get_field_definition( field, field_name=es_unique_id_code_field_name( unique_id_field_name))) if isinstance(field, FieldSet): if field.is_group(): self._get_submission_fields(fields_definition, field.fields, field.code) else: es_field_name = es_questionnaire_field_name( field.code, self.latest_form_model.id, parent_field_name) fields_definition.append( get_field_definition_with_binary_type( field, field_name=es_field_name)) continue fields_definition.append( get_field_definition(field, field_name=es_questionnaire_field_name( field.code, self.latest_form_model.id, parent_field_name)))
def create_response(self, required_field_names, query): entity_question_codes = [ es_field_name(field.code, self.form_model.id) for field in self.form_model.entity_questions ] meta_fields = [SubmissionIndexConstants.DATASENDER_ID_KEY] meta_fields.extend([ es_unique_id_code_field_name(code) for code in entity_question_codes ]) submissions = [] language = get_language() for res in query.values_dict(tuple(required_field_names)): submission = [res._id] for key in required_field_names: if not key in meta_fields: if key in entity_question_codes: self.combine_name_and_id(short_code=res.get( es_unique_id_code_field_name(key)), entity_name=res.get(key), submission=submission) elif key == SubmissionIndexConstants.DATASENDER_NAME_KEY: self.combine_name_and_id( res.get( SubmissionIndexConstants.DATASENDER_ID_KEY), res.get( SubmissionIndexConstants.DATASENDER_NAME_KEY), submission) elif key == 'status': submission.append(ugettext(res.get(key))) elif key == 'error_msg': error_msg = res.get(key) if error_msg.find('| |') != -1: error_msg = error_msg.split('| |,')[[ 'en', 'fr' ].index(language)] submission.append(error_msg) else: submission.append(res.get(key)) submissions.append(submission) return submissions
def get_column_title(self): header = HeaderFactory(self._form_model).create_header(self.submission_type) header_dict = header.get_header_field_dict() header_dict.pop('ds_id', None) unique_question_field_names = [es_unique_id_code_field_name(es_questionnaire_field_name(field.code, self._form_model.id, field.parent_field_code)) for field in self._form_model.entity_questions] for field_name in unique_question_field_names: header_dict.pop(field_name, None) return header_dict.values()
def get_field_names_as_header_name(self): headers = self.get_header_dict().keys() entity_questions = self.form_model.entity_questions for entity_question in entity_questions: if not self.form_model.is_part_of_repeat_field(entity_question): headers.remove( es_unique_id_code_field_name( es_questionnaire_field_name( entity_question.code, self.form_model.id, entity_question.parent_field_code))) return headers
def add_unique_id_field(self, unique_id_field, header_dict): unique_id_question_code = unique_id_field.code subject_title = unique_id_field.unique_id_type unique_id_field_name = es_questionnaire_field_name( unique_id_question_code, self.form_model.id, unique_id_field.parent_field_code) header_dict.update({unique_id_field_name: unique_id_field.label}) header_dict.update({ es_unique_id_code_field_name(unique_id_field_name): "%s ID" % subject_title })
def _add_unique_id_filters(self, query, uniqueIdFilters): if uniqueIdFilters: for uniqueIdType, uniqueIdFilter in uniqueIdFilters.iteritems(): if uniqueIdFilter: search_options = elasticutils.F() for question in [ question for question in self.form_model.entity_questions if question.unique_id_type == uniqueIdType ]: es_field_code = es_unique_id_code_field_name( es_field_name(question.code, self.form_model.id)) search_options |= elasticutils.F( **{es_field_code: uniqueIdFilter}) query = query.filter(search_options) return query
def _add_unique_id_filters(form_model, unique_id_filters, search): if unique_id_filters: for uniqueIdType, uniqueIdFilter in unique_id_filters.iteritems(): if uniqueIdFilter: unique_id_filters = [] for question in [ question for question in form_model.entity_questions if question.unique_id_type == uniqueIdType ]: es_field_code = es_unique_id_code_field_name( es_questionnaire_field_name( question.code, form_model.id, parent_field_code=question.parent_field_code) ) + "_exact" unique_id_filters.append( F("term", **{es_field_code: uniqueIdFilter})) search = search.filter(F('or', unique_id_filters)) return search
def get_mappings(self): fields_definition = [] fields_definition.extend(get_submission_meta_fields()) for field in self.latest_form_model.fields: if isinstance(field, UniqueIdField): unique_id_field_name = es_field_name(field.code, self.latest_form_model.id) fields_definition.append( get_field_definition( field, field_name=es_unique_id_code_field_name( unique_id_field_name))) fields_definition.append( get_field_definition(field, field_name=es_field_name( field.code, self.latest_form_model.id))) mapping = self.get_fields_mapping_by_field_def( doc_type=self.latest_form_model.id, fields_definition=fields_definition) return mapping
def _update_with_form_model_fields(dbm, submission_doc, search_dict, form_model): #Submission value may have capitalized keys in some cases. This conversion is to do #case insensitive lookup. submission_values = OrderedDict( (k.lower(), v) for k, v in submission_doc.values.iteritems()) for field in form_model.fields: entry = submission_values.get(lower(field.code)) if field.is_entity_field: if entry: original_field = form_model.get_field_by_code_and_rev( field.code, submission_doc.form_model_revision) if is_original_question_changed_from_choice_answer_type( original_field, field): entry = convert_choice_options_to_options_text( original_field, entry) entity_name = lookup_entity_name(dbm, entry, [field.unique_id_type]) entry_code = entry search_dict.update({ es_unique_id_code_field_name( es_field_name(lower(field.code), form_model.id)): entry_code or UNKNOWN }) entry = entity_name elif field.type == "select": field = _update_select_field_by_revision(field, form_model, submission_doc) if field.type == "select": entry = field.get_option_value_list(entry) elif field.type == "select1": entry = ",".join(field.get_option_value_list(entry)) elif field.type == "select1": field = _update_select_field_by_revision(field, form_model, submission_doc) if field.type == "select": entry = field.get_option_value_list(entry) elif field.type == "select1": entry = ",".join(field.get_option_value_list(entry)) elif field.type == 'text': field_for_revision = form_model.get_field_by_code_and_rev( field.code, submission_doc.form_model_revision) if isinstance(field_for_revision, SelectField): entry = _get_select_field_answer_from_snapshot( entry, field_for_revision) elif field.type == "date": try: if form_model.revision != submission_doc.form_model_revision: old_submission_value = entry to_format = field.date_format field_for_revision = form_model.get_field_by_code_and_rev( field.code, submission_doc.form_model_revision) if isinstance(field_for_revision, DateField): current_date = field_for_revision.__date__(entry) entry = current_date.strftime( DateField.DATE_DICTIONARY.get(to_format)) elif isinstance(field_for_revision, SelectField): entry = _get_select_field_answer_from_snapshot( entry, field_for_revision) logger.info( "Converting old date submission from %s to %s" % (old_submission_value, entry)) except Exception as ignore_conversion_errors: pass if entry: search_dict.update( {es_field_name(lower(field.code), form_model.id): entry}) search_dict.update({'void': submission_doc.void}) return search_dict
def _update_search_dict(dbm, form_model, fields, search_dict, submission_doc, submission_values, parent_field_name=None): for field in fields: field_code = field.code entry = submission_values.get(field_code) if field.is_entity_field: if entry: original_field = form_model.get_field_by_code_and_rev(field.code, submission_doc.form_model_revision) if is_original_question_changed_from_choice_answer_type(original_field, field): entry = convert_choice_options_to_options_text(original_field, entry) entity_name = lookup_entity_name(dbm, entry, [field.unique_id_type]) entry_code = entry search_dict.update( {es_unique_id_code_field_name( es_questionnaire_field_name(field.code, form_model.id, parent_field_name)): entry_code or UNKNOWN}) entry = entity_name elif field.type == "select": field = _get_select_field_by_revision(field, form_model, submission_doc) if field.type == "select": entry = field.get_option_value_list(entry) elif field.type == "select1": entry = ",".join(field.get_option_value_list(entry)) elif field.type == "select1": field = _get_select_field_by_revision(field, form_model, submission_doc) if field.type == "select": entry = field.get_option_value_list(entry) elif field.type == "select1": entry = ",".join(field.get_option_value_list(entry)) elif field.type == 'text': field_for_revision = form_model.get_field_by_code_and_rev(field.code, submission_doc.form_model_revision) if isinstance(field_for_revision, SelectField): entry = _get_select_field_answer_from_snapshot(entry, field_for_revision) elif field.type == "date": try: if form_model.revision != submission_doc.form_model_revision: old_submission_value = entry to_format = field.date_format field_for_revision = form_model.get_field_by_code_and_rev(field.code, submission_doc.form_model_revision) if isinstance(field_for_revision, DateField): current_date = field_for_revision.__date__(entry) entry = current_date.strftime(DateField.DATE_DICTIONARY.get(to_format)) elif isinstance(field_for_revision, SelectField): entry = _get_select_field_answer_from_snapshot(entry, field_for_revision) logger.info("Converting old date submission from %s to %s" % (old_submission_value, entry)) except Exception as ignore_conversion_errors: pass if entry: if isinstance(field, FieldSet): if field.is_group(): for value in submission_values[field_code]: _update_search_dict(dbm, form_model, field.fields, search_dict, submission_doc, value, field.code) else: _update_repeat_fields_with_choice_values(entry, field) _update_name_unique_code(dbm, entry, field) search_dict.update( {es_questionnaire_field_name(field_code, form_model.id, parent_field_name): json.dumps(entry)}) else: search_dict.update({es_questionnaire_field_name(field.code, form_model.id, parent_field_name): entry}) search_dict.update({'void': submission_doc.void}) search_dict.update({'is_anonymous': submission_doc.is_anonymous_submission})
def _update_search_dict(dbm, form_model, fields, search_dict, submission_doc, submission_values, parent_field_name=None): for field in fields: entry = submission_values.get(field.code) label_to_be_displayed = get_label_to_be_displayed( entry, field, form_model, submission_doc) es_field_name = es_questionnaire_field_name(field.code, form_model.id, parent_field_name) if label_to_be_displayed: if 'media' not in search_dict.keys(): search_dict.update({'media': {}}) if isinstance(field, PhotoField): search_dict['media'].update({ es_field_name: { 'type': 'image', 'value': entry, 'download_link': '/download/attachment/' + submission_doc.id + '/' + entry, 'preview_link': '/download/attachment/' + submission_doc.id + '/preview_' + entry, } }) if isinstance(field, AudioField): search_dict['media'].update({ es_field_name: { 'type': 'audio', 'value': entry, 'download_link': '/download/attachment/' + submission_doc.id + '/' + entry, } }) if isinstance(field, VideoField): search_dict['media'].update({ es_field_name: { 'type': 'video', 'value': entry, 'download_link': '/download/attachment/' + submission_doc.id + '/' + entry, } }) if isinstance(field, FieldSet): if field.is_group(): for value in entry: _update_search_dict(dbm, form_model, field.fields, search_dict, submission_doc, value, field.code) else: _update_repeat_fields_with_choice_values( label_to_be_displayed, field) _update_name_unique_code(dbm, label_to_be_displayed, field) search_dict.update( {es_field_name: json.dumps(label_to_be_displayed)}) elif field.is_entity_field: entity, choice_to_entity_entry = get_entity( dbm, entry, field, form_model, submission_doc) if entry: search_dict.update({ es_unique_id_details_field_name(es_field_name): entity }) search_dict.update({ es_unique_id_code_field_name(es_field_name): choice_to_entity_entry or UNKNOWN }) search_dict.update({es_field_name: entry and entity.get('q2')}) else: search_dict.update({es_field_name: label_to_be_displayed}) search_dict.update({'void': submission_doc.void}) search_dict.update( {'is_anonymous': submission_doc.is_anonymous_submission})