def before_index(self, search_data): if not self.is_supported_package_type(search_data): return search_data extract_title = LangToString('title') validated_dict = json.loads(search_data['validated_data_dict']) # log.debug(pprint.pformat(validated_dict)) search_data['res_name'] = [extract_title(r) for r in validated_dict[u'resources']] # noqa search_data['res_description'] = [LangToString('description')(r) for r in validated_dict[u'resources']] # noqa search_data['res_format'] = self._prepare_formats_for_index(validated_dict[u'resources']) # noqa search_data['res_rights'] = [simplify_terms_of_use(r['rights']) for r in validated_dict[u'resources']] # noqa search_data['title_string'] = extract_title(validated_dict) search_data['description'] = LangToString('description')(validated_dict) # noqa if 'political_level' in validated_dict[u'organization']: search_data['political_level'] = validated_dict[u'organization'][u'political_level'] # noqa try: # index language-specific values (or it's fallback) text_field_items = {} for lang_code in get_langs(): search_data['title_' + lang_code] = get_localized_value( validated_dict['title'], lang_code ) search_data['title_string_' + lang_code] = munge_title_to_name( get_localized_value(validated_dict['title'], lang_code) ) search_data['description_' + lang_code] = get_localized_value( validated_dict['description'], lang_code ) search_data['keywords_' + lang_code] = get_localized_value( validated_dict['keywords'], lang_code ) text_field_items['text_' + lang_code] = [get_localized_value(validated_dict['description'], lang_code)] # noqa text_field_items['text_' + lang_code].extend(search_data['keywords_' + lang_code]) # noqa text_field_items['text_' + lang_code].extend([r['title'][lang_code] for r in validated_dict['resources'] if r['title'][lang_code]]) # noqa text_field_items['text_' + lang_code].extend([r['description'][lang_code] for r in validated_dict['resources'] if r['description'][lang_code]]) # noqa # flatten values for text_* fields for key, value in text_field_items.iteritems(): search_data[key] = ' '.join(value) except KeyError: pass # log.debug(pprint.pformat(search_data)) return search_data
def before_index(self, search_data): if not self.is_supported_package_type(search_data): return search_data extract_title = LangToString('title') validated_dict = json.loads(search_data['validated_data_dict']) search_data['res_name'] = [extract_title(r) for r in validated_dict[u'resources']] # noqa search_data['res_description'] = [LangToString('description')(r) for r in validated_dict[u'resources']] # noqa search_data['res_format'] = self._prepare_formats_for_index(validated_dict[u'resources']) # noqa search_data['res_rights'] = [simplify_terms_of_use(r['rights']) for r in validated_dict[u'resources']] # noqa search_data['title_string'] = extract_title(validated_dict) search_data['description'] = LangToString('description')(validated_dict) # noqa if 'political_level' in validated_dict[u'organization']: search_data['political_level'] = validated_dict[u'organization'][u'political_level'] # noqa try: # index language-specific values (or it's fallback) text_field_items = {} for lang_code in get_langs(): search_data['title_' + lang_code] = get_localized_value( validated_dict['title'], lang_code ) search_data['title_string_' + lang_code] = munge_title_to_name( get_localized_value(validated_dict['title'], lang_code) ) search_data['description_' + lang_code] = get_localized_value( validated_dict['description'], lang_code ) search_data['keywords_' + lang_code] = get_localized_value( validated_dict['keywords'], lang_code ) text_field_items['text_' + lang_code] = [get_localized_value(validated_dict['description'], lang_code)] # noqa text_field_items['text_' + lang_code].extend(search_data['keywords_' + lang_code]) # noqa text_field_items['text_' + lang_code].extend([r['title'][lang_code] for r in validated_dict['resources'] if r['title'][lang_code]]) # noqa text_field_items['text_' + lang_code].extend([r['description'][lang_code] for r in validated_dict['resources'] if r['description'][lang_code]]) # noqa # flatten values for text_* fields for key, value in text_field_items.iteritems(): search_data[key] = ' '.join(value) except KeyError: pass return search_data
def _extract_lang_value(self, value, lang_code): new_value = sh.parse_json(value) if isinstance(new_value, dict): return sh.get_localized_value(new_value, lang_code, default_value='') return value
def test_get_localized_value_dict(self): lang_dict = { 'de': 'DE value', 'fr': 'FR value', 'it': 'IT value', 'en': 'EN value', } result = helpers.get_localized_value(lang_dict, 'de') self.assertEquals(lang_dict['de'], result)
def test_get_localized_value_fallback(self): lang_dict = { 'de': 'DE value', 'fr': 'FR value', 'it': 'IT value', 'en': '', } result = helpers.get_localized_value(lang_dict, 'en') # if en does not exist, fallback to de self.assertEquals(lang_dict['de'], result)
def test_get_localized_value_no_lang(self, mock_request): mock_request.environ = {'CKAN_LANG': 'fr'} lang_dict = { 'de': 'DE value', 'fr': 'FR value', 'it': 'IT value', 'en': 'EN value', } result = helpers.get_localized_value(lang_dict) self.assertEquals(lang_dict['fr'], result)
def test_get_localized_value_invalid_dict(self): test_dict = {"test": "dict"} result = helpers.get_localized_value(test_dict) self.assertEquals(test_dict, result)
def test_get_localized_value_no_lang(self, mock_request): mock_request.environ = {"CKAN_LANG": "fr"} lang_dict = {"de": "DE value", "fr": "FR value", "it": "IT value", "en": "EN value"} result = helpers.get_localized_value(lang_dict) self.assertEquals(lang_dict["fr"], result)
def test_get_localized_value_fallback(self): lang_dict = {"de": "DE value", "fr": "FR value", "it": "IT value", "en": ""} result = helpers.get_localized_value(lang_dict, "en") # if en does not exist, fallback to de self.assertEquals(lang_dict["de"], result)
def test_get_localized_value_dict(self): lang_dict = {"de": "DE value", "fr": "FR value", "it": "IT value", "en": "EN value"} result = helpers.get_localized_value(lang_dict, "de") self.assertEquals(lang_dict["de"], result)
def before_index(self, search_data): if not self.is_supported_package_type(search_data): return search_data extract_title = LangToString('title') validated_dict = json.loads(search_data['validated_data_dict']) search_data['res_name'] = [ extract_title(r) for r in validated_dict[u'resources'] ] # noqa search_data['res_name_en'] = [ sh.get_localized_value(r['title'], 'en') for r in validated_dict[u'resources'] ] # noqa search_data['res_name_de'] = [ sh.get_localized_value(r['title'], 'de') for r in validated_dict[u'resources'] ] # noqa search_data['res_name_fr'] = [ sh.get_localized_value(r['title'], 'fr') for r in validated_dict[u'resources'] ] # noqa search_data['res_name_it'] = [ sh.get_localized_value(r['title'], 'it') for r in validated_dict[u'resources'] ] # noqa search_data['res_description_en'] = [ sh.get_localized_value(r['description'], 'en') for r in validated_dict[u'resources'] ] # noqa search_data['res_description_de'] = [ sh.get_localized_value(r['description'], 'de') for r in validated_dict[u'resources'] ] # noqa search_data['res_description_fr'] = [ sh.get_localized_value(r['description'], 'fr') for r in validated_dict[u'resources'] ] # noqa search_data['res_description_it'] = [ sh.get_localized_value(r['description'], 'it') for r in validated_dict[u'resources'] ] # noqa search_data['groups_en'] = [ sh.get_localized_value(g['display_name'], 'en') for g in validated_dict[u'groups'] ] # noqa search_data['groups_de'] = [ sh.get_localized_value(g['display_name'], 'de') for g in validated_dict[u'groups'] ] # noqa search_data['groups_fr'] = [ sh.get_localized_value(g['display_name'], 'fr') for g in validated_dict[u'groups'] ] # noqa search_data['groups_it'] = [ sh.get_localized_value(g['display_name'], 'it') for g in validated_dict[u'groups'] ] # noqa search_data['res_description'] = [ LangToString('description')(r) for r in validated_dict[u'resources'] ] # noqa search_data['res_format'] = self._prepare_formats_for_index( validated_dict[u'resources']) # noqa search_data['res_rights'] = [ sh.simplify_terms_of_use(r['rights']) for r in validated_dict[u'resources'] ] # noqa search_data['title_string'] = extract_title(validated_dict) search_data['description'] = LangToString('description')( validated_dict) # noqa if 'political_level' in validated_dict[u'organization']: search_data['political_level'] = validated_dict[u'organization'][ u'political_level'] # noqa search_data['identifier'] = validated_dict.get('identifier') search_data['contact_points'] = [ c['name'] for c in validated_dict.get('contact_points', []) ] # noqa search_data['publishers'] = [ p['label'] for p in validated_dict.get('publishers', []) ] # noqa # TODO: Remove the try-except-block. # This fixes the index while we have 'wrong' relations on # datasets harvested with an old version of ckanext-geocat try: search_data['see_alsos'] = [ d['dataset_identifier'] for d in validated_dict.get('see_alsos', []) ] # noqa except TypeError: search_data['see_alsos'] = [ d for d in validated_dict.get('see_alsos', []) ] # noqa # make sure we're not dealing with NoneType if search_data['metadata_created'] is None: search_data['metadata_created'] = '' if search_data['metadata_modified'] is None: search_data['metadata_modified'] = '' try: # index language-specific values (or it's fallback) for lang_code in sh.get_langs(): search_data['title_' + lang_code] = sh.get_localized_value( validated_dict['title'], lang_code) search_data['title_string_' + lang_code] = munge_title_to_name( sh.get_localized_value(validated_dict['title'], lang_code)) search_data['description_' + lang_code] = sh.get_localized_value( # noqa validated_dict['description'], lang_code) search_data['keywords_' + lang_code] = sh.get_localized_value( validated_dict['keywords'], lang_code) search_data['organization_' + lang_code] = sh.get_localized_value( # noqa validated_dict['organization']['title'], lang_code) except KeyError: pass # clean terms for suggest context search_data = self._prepare_suggest_context(search_data, validated_dict) return search_data
def _extract_lang_value(self, value, lang_code): new_value = parse_json(value) if isinstance(new_value, dict): return get_localized_value(new_value, lang_code, default_value='') return value
def test_get_localized_value_invalid_dict(self): test_dict = {'test': 'dict'} result = helpers.get_localized_value(test_dict) self.assertEquals(test_dict, result)