def update_submission_search_for_subject_edition(entity_doc, dbm): from datawinners.search.submission_query import SubmissionQueryBuilder entity_type = entity_doc.entity_type projects = [] for row in dbm.load_all_rows_in_view('projects_by_subject_type', key=entity_type[0], include_docs=True): projects.append( Project.new_from_doc(dbm, ProjectDocument.wrap(row['doc']))) for project in projects: entity_field_code = None for field in project.entity_questions: if [field.unique_id_type] == entity_type: entity_field_code = field.code if entity_field_code: unique_id_field_name = es_field_name(entity_field_code, project.id) fields_mapping = { unique_id_field_name: entity_doc.data['name']['value'] } args = { es_unique_id_code_field_name(unique_id_field_name): entity_doc.short_code } survey_response_filtered_query = SubmissionQueryBuilder( project).query_all(dbm.database_name, project.id, **args) for survey_response in survey_response_filtered_query.all(): SubmissionIndexUpdateHandler( dbm.database_name, project.id).update_field_in_submission_index( survey_response._id, fields_mapping)
def get_headers(self, user, subject_type): manager = get_database_manager(user) form_model = get_form_model_by_entity_type(manager, [subject_type]) #return header_fields(form_model,key_attribute='code').keys() subject_es_headers = [] for code in header_fields(form_model, key_attribute='code').keys(): subject_es_headers.append(es_field_name(code, form_model.id)) return subject_es_headers
def get_mappings(self): fields_definition = [] fields_definition.extend(get_submission_meta_fields()) for field in self.form_model.fields: fields_definition.append( get_field_definition(field, field_name=es_field_name(field.code, self.form_model.id))) mapping = self.get_fields_mapping_by_field_def(doc_type=self.form_model.id, fields_definition=fields_definition) return mapping
def test_submission_status_headers_for_errored_submissions(self): self.form_model.fields = [self.field1, self.field2, self.field3, self.field4] self.form_model.entity_questions = [self.field3, self.field4] headers = ErroredSubmissionHeader(self.form_model).get_header_field_names() expected = [es_field_name(f, self.form_model.id) for f in ["ds_id", "ds_name", "date","error_msg", "q1", "q2", "q3", "q3_unique_code", "q4", "q4_unique_code"]] self.assertListEqual(expected, headers)
def test_should_return_submission_log_specific_header_fields(self): self.form_model.fields = [self.field1, self.field2, self.field3, self.field4] self.form_model.entity_questions = [self.field3, self.field4] headers = AllSubmissionHeader(self.form_model).get_header_field_names() expected = [es_field_name(f, self.form_model.id) for f in ["ds_id", "ds_name", "date", "status", "q1", "q2", "q3", "q3_unique_code", "q4", "q4_unique_code"]] self.assertListEqual(expected, headers)
def update_static_header_info(self): header_dict = OrderedDict() header_dict.update( {SubmissionIndexConstants.DATASENDER_ID_KEY: "Datasender Id"}) header_dict.update( {SubmissionIndexConstants.DATASENDER_NAME_KEY: "Datasender Name"}) header_dict.update({"date": "Submission Date"}) subject_title = self.form_model.entity_type[0].title() header_dict.update({ es_field_name(self.form_model.entity_question.code, self.form_model.id): subject_title }) header_dict.update({'entity_short_code': "%s ID" % subject_title}) if self.form_model.event_time_question: header_dict.update({ es_field_name(self.form_model.event_time_question.code, self.form_model.id): "Reporting Date" }) return header_dict
def get_header_dict(self): header_dict = OrderedDict() header_dict.update(self.update_static_header_info()) def key_attribute(field): return field.code.lower() headers = header_fields(self.form_model, key_attribute) for field_code, val in headers.items(): key = es_field_name(field_code, self.form_model.id) if not header_dict.has_key(key): header_dict.update({key: val}) if self.form_model.is_entity_type_reporter(): # For summary projects there will be an extra datasender info question(code eid). # This condition removes that extra question. header_dict.pop( es_field_name(self.form_model.entity_question.code, self.form_model.id)) header_dict.pop('entity_short_code') return header_dict
def create_subject_mapping(dbm, form_model): es = get_elasticsearch_handle() fields_definition = [] for field in form_model.fields: fields_definition.append( get_field_definition(field, field_name=es_field_name( field.code, form_model.id))) mapping = get_fields_mapping_by_field_def( doc_type=form_model.id, fields_definition=fields_definition) es.put_mapping(dbm.database_name, form_model.entity_type[0], mapping)
def get_mappings(self): fields_definition = [] fields_definition.extend(get_submission_meta_fields()) for field in self.latest_form_model.fields: if isinstance(field, UniqueIdField): unique_id_field_name = es_field_name(field.code, self.latest_form_model.id) fields_definition.append( get_field_definition( field, field_name=es_unique_id_code_field_name( unique_id_field_name))) fields_definition.append( get_field_definition(field, field_name=es_field_name( field.code, self.latest_form_model.id))) mapping = self.get_fields_mapping_by_field_def( doc_type=self.latest_form_model.id, fields_definition=fields_definition) return mapping
def get_column_title(self): header = HeaderFactory(self._form_model).create_header( self.submission_type) header_dict = header.get_header_field_dict() header_dict.pop('ds_id', None) unique_question_field_names = [ es_unique_id_code_field_name( es_field_name(field.code, self._form_model.id)) for field in self._form_model.entity_questions ] for field_name in unique_question_field_names: header_dict.pop(field_name, None) return header_dict.values()
def test_submission_status_headers_for_success_and_erred_submissions(self): form_model = MagicMock(spec=FormModel, id="2323") form_model.is_entity_type_reporter.return_value = False entity_question_field = Mock(spec=Field) form_model.entity_question = entity_question_field form_model.event_time_question = None entity_question_field.code.lower.return_value = 'eid' with patch("datawinners.search.submission_headers.header_fields") as header_fields: header_fields.return_value = {} query_params = {"filter": "success"} headers = SubmissionQuery(form_model, query_params).get_headers(Mock, "code") expected = [es_field_name(f, "2323") for f in ["ds_id", "ds_name", "date", "eid", "entity_short_code"]] self.assertListEqual(expected, headers) query_params = {"filter": "error"} headers = SubmissionQuery(form_model, query_params).get_headers(Mock, "code") expected = [es_field_name(f, "2323") for f in ["ds_id", "ds_name", "date", "error_msg", "eid", "entity_short_code"]] self.assertListEqual(expected, headers)
def subject_autocomplete(request, entity_type): search_text = lower(request.GET["term"] or "") database_name = get_database_name(request.user) dbm = get_database_manager(request.user) form_model = get_form_model_by_entity_type(dbm, [entity_type.lower()]) subject_name_field = get_field_by_attribute_value(form_model, 'name', 'name') es_field_name_for_subject_name = es_field_name(subject_name_field.code, form_model.id) subject_short_code_field = get_field_by_attribute_value( form_model, 'name', 'short_code') es_field_name_for_short_code = es_field_name(subject_short_code_field.code, form_model.id) query = elasticutils.S().es(urls=ELASTIC_SEARCH_URL).indexes(database_name).doctypes(lower(entity_type)) \ .query(or_={es_field_name_for_subject_name + '__match': search_text, es_field_name_for_subject_name + '_value': search_text, es_field_name_for_short_code + '__match': search_text, es_field_name_for_short_code + '_value': search_text}) \ .values_dict() resp = [{ "id": r[es_field_name_for_short_code], "label": r[es_field_name_for_subject_name] } for r in query[:min(query.count(), 50)]] return HttpResponse(json.dumps(resp))
def _add_unique_id_filters(self, query, uniqueIdFilters): if uniqueIdFilters: for uniqueIdType, uniqueIdFilter in uniqueIdFilters.iteritems(): if uniqueIdFilter: search_options = elasticutils.F() for question in [ question for question in self.form_model.entity_questions if question.unique_id_type == uniqueIdType ]: es_field_code = es_unique_id_code_field_name( es_field_name(question.code, self.form_model.id)) search_options |= elasticutils.F( **{es_field_code: uniqueIdFilter}) query = query.filter(search_options) return query
def test_should_return_submission_log_specific_header_fields(self): form_model = MagicMock(spec=FormModel, id="2323") #form_model.entity_type = ["clinic"] entity_question_field = Mock(spec=Field) form_model.entity_question = entity_question_field form_model.event_time_question = None form_model.is_entity_type_reporter.return_value = False entity_question_field.code.lower.return_value = 'eid' with patch("datawinners.search.submission_headers.header_fields") as header_fields: header_fields.return_value = {} query_params = {"filter": "all"} headers = SubmissionQuery(form_model, query_params).get_headers() expected = [es_field_name(f, "2323") for f in ["ds_id", "ds_name", "date", "status", "eid", "entity_short_code"]] self.assertListEqual(expected, headers)
def update_submission_search_for_subject_edition(entity_doc, dbm): from datawinners.search.submission_query import SubmissionQueryBuilder entity_type = entity_doc.entity_type projects = dbm.load_all_rows_in_view('projects_by_subject_type', key=entity_type[0], include_docs=True) form_models = _get_form_models_from_projects(dbm, projects) for form_model in form_models: entity_field_name = lower(form_model.entity_question.code) fields_mapping = {es_field_name(entity_field_name, form_model.id): entity_doc.data['name']['value']} args = {'entity_short_code_value': entity_doc.short_code} survey_response_filtered_query = SubmissionQueryBuilder(form_model).query_all(dbm.database_name, form_model.id, **args) for survey_response in survey_response_filtered_query.all(): SubmissionIndexUpdateHandler(dbm.database_name, form_model.id).update_field_in_submission_index( survey_response._id, fields_mapping)
def create_response(self, required_field_names, query): entity_question_codes = [ es_field_name(field.code, self.form_model.id) for field in self.form_model.entity_questions ] meta_fields = [SubmissionIndexConstants.DATASENDER_ID_KEY] meta_fields.extend([ es_unique_id_code_field_name(code) for code in entity_question_codes ]) submissions = [] language = get_language() for res in query.values_dict(tuple(required_field_names)): submission = [res._id] for key in required_field_names: if not key in meta_fields: if key in entity_question_codes: self.combine_name_and_id(short_code=res.get( es_unique_id_code_field_name(key)), entity_name=res.get(key), submission=submission) elif key == SubmissionIndexConstants.DATASENDER_NAME_KEY: self.combine_name_and_id( res.get( SubmissionIndexConstants.DATASENDER_ID_KEY), res.get( SubmissionIndexConstants.DATASENDER_NAME_KEY), submission) elif key == 'status': submission.append(ugettext(res.get(key))) elif key == 'error_msg': error_msg = res.get(key) if error_msg.find('| |') != -1: error_msg = error_msg.split('| |,')[[ 'en', 'fr' ].index(language)] submission.append(error_msg) else: submission.append(res.get(key)) submissions.append(submission) return submissions
def test_should_have_reporting_date_header_if_form_model_has_reporting_date(self): form_model = MagicMock(spec=FormModel, id="2323") form_model.is_entity_type_reporter.return_value = False entity_question_field = Mock(spec=Field) form_model.entity_question = entity_question_field entity_question_field.code.lower.return_value = 'eid' event_time_field = Mock(spec=Field) form_model.event_time_question = event_time_field event_time_field.code.lower.return_value = "rp_date" with patch("datawinners.search.submission_headers.header_fields") as header_fields: header_fields.return_value = {} query_params = {"filter": "success"} headers = SubmissionQuery(form_model, query_params).get_headers(Mock, "code") expected = [es_field_name(f, "2323") for f in ["ds_id", "ds_name", "date", "eid", "entity_short_code", "rp_date"]] self.assertListEqual(expected, headers)
def get_facet_response_for_choice_fields(query_with_criteria, choice_fields, form_model_id): facet_results = [] for field in choice_fields: field_name = es_field_name(field.code, form_model_id) + "_exact" facet_response = query_with_criteria.facet(field_name, filtered=True).facet_counts() facet_result_options = [] facet_result = { "es_field_name": field_name, "facets": facet_result_options, # find total submissions containing specified answer "total": query_with_criteria.filter(~F(**{field_name: None})).count() } for option, facet_list in facet_response.iteritems(): for facet in facet_list: facet_result_options.append({ "term": facet['term'], "count": facet['count'] }) facet_results.append(facet_result) return facet_results
def _update_with_form_model_fields(dbm, submission_doc, search_dict, form_model): #Submission value may have capitalized keys in some cases. This conversion is to do #case insensitive lookup. submission_values = OrderedDict((k.lower(), v) for k,v in submission_doc.values.iteritems()) for field in form_model.fields: entry = submission_values.get(lower(field.code)) if field.is_entity_field: entity_name = lookup_entity_name(dbm, entry, form_model.entity_type) entry_code = UNKNOWN if entity_name == UNKNOWN else entry search_dict.update({"entity_short_code": entry_code or UNKNOWN}) entry = entity_name elif field.type == "select": field = _update_select_field_by_revision(field, form_model, submission_doc) if field.type == "select": entry = field.get_option_value_list(entry) elif field.type == "select1": entry = ",".join(field.get_option_value_list(entry)) elif field.type == "select1": field = _update_select_field_by_revision(field, form_model, submission_doc) if field.type == "select": entry = field.get_option_value_list(entry) elif field.type == "select1": entry = ",".join(field.get_option_value_list(entry)) elif field.type == "date": try: if form_model.revision != submission_doc.form_model_revision: old_submission_value = entry to_format = field.date_format current_format = form_model.get_field_by_code_and_rev(field.code, submission_doc.form_model_revision).__date__(entry) entry = current_format.strftime(DateField.DATE_DICTIONARY.get(to_format)) logger.info("Converting old date submission from %s to %s" % (old_submission_value, entry)) except Exception as ignore_conversion_errors: pass if entry: search_dict.update({es_field_name(lower(field.code), form_model.id): entry}) search_dict.update({'void': submission_doc.void}) return search_dict
def get_columns(self): header = HeaderFactory(self._form_model).create_header( self.submission_type) header_dict = header.get_header_field_dict() # header_dict = OrderedDict({'name':"Name"}, {"p":"Place", "}) if self._form_model.entity_type == ["reporter"]: header_dict.pop("entity_short_code", None) result = OrderedDict() for key in header_dict: if key != SubmissionIndexConstants.DATASENDER_ID_KEY: result.update({key: {"label": header_dict[key]}}) if key == SubmissionIndexConstants.DATASENDER_NAME_KEY: #add key column after name self.add_datasender_id_column(header_dict, result) for field in self._form_model.fields: field_name = es_field_name(field.code.lower(), self._form_model.id) if result.has_key(field_name): result.get(field_name).update({"type": field.type}) if field.type == "date": result.get(field_name).update( {"format": field.date_format}) return result
def _update_with_form_model_fields(dbm, submission_doc, search_dict, form_model): #Submission value may have capitalized keys in some cases. This conversion is to do #case insensitive lookup. submission_values = OrderedDict( (k.lower(), v) for k, v in submission_doc.values.iteritems()) for field in form_model.fields: entry = submission_values.get(lower(field.code)) if field.is_entity_field: if entry: original_field = form_model.get_field_by_code_and_rev( field.code, submission_doc.form_model_revision) if is_original_question_changed_from_choice_answer_type( original_field, field): entry = convert_choice_options_to_options_text( original_field, entry) entity_name = lookup_entity_name(dbm, entry, [field.unique_id_type]) entry_code = entry search_dict.update({ es_unique_id_code_field_name( es_field_name(lower(field.code), form_model.id)): entry_code or UNKNOWN }) entry = entity_name elif field.type == "select": field = _update_select_field_by_revision(field, form_model, submission_doc) if field.type == "select": entry = field.get_option_value_list(entry) elif field.type == "select1": entry = ",".join(field.get_option_value_list(entry)) elif field.type == "select1": field = _update_select_field_by_revision(field, form_model, submission_doc) if field.type == "select": entry = field.get_option_value_list(entry) elif field.type == "select1": entry = ",".join(field.get_option_value_list(entry)) elif field.type == 'text': field_for_revision = form_model.get_field_by_code_and_rev( field.code, submission_doc.form_model_revision) if isinstance(field_for_revision, SelectField): entry = _get_select_field_answer_from_snapshot( entry, field_for_revision) elif field.type == "date": try: if form_model.revision != submission_doc.form_model_revision: old_submission_value = entry to_format = field.date_format field_for_revision = form_model.get_field_by_code_and_rev( field.code, submission_doc.form_model_revision) if isinstance(field_for_revision, DateField): current_date = field_for_revision.__date__(entry) entry = current_date.strftime( DateField.DATE_DICTIONARY.get(to_format)) elif isinstance(field_for_revision, SelectField): entry = _get_select_field_answer_from_snapshot( entry, field_for_revision) logger.info( "Converting old date submission from %s to %s" % (old_submission_value, entry)) except Exception as ignore_conversion_errors: pass if entry: search_dict.update( {es_field_name(lower(field.code), form_model.id): entry}) search_dict.update({'void': submission_doc.void}) return search_dict
def get_date_field_name(self): return es_field_name(self.reporting_date_field.code, self.form_model.id)
def get_date_field_name(self): return es_field_name(ES_SUBMISSION_FIELD_DATE, None)