def before_index(self, search_data):
if not self.is_supported_package_type(search_data):
return search_data
extract_title = LangToString('title')
validated_dict = json.loads(search_data['validated_data_dict'])
# log.debug(pprint.pformat(validated_dict))
search_data['res_name'] = [extract_title(r) for r in validated_dict[u'resources']] # noqa
search_data['res_description'] = [LangToString('description')(r) for r in validated_dict[u'resources']] # noqa
search_data['res_format'] = self._prepare_formats_for_index(validated_dict[u'resources']) # noqa
search_data['res_rights'] = [simplify_terms_of_use(r['rights']) for r in validated_dict[u'resources']] # noqa
search_data['title_string'] = extract_title(validated_dict)
search_data['description'] = LangToString('description')(validated_dict) # noqa
if 'political_level' in validated_dict[u'organization']:
search_data['political_level'] = validated_dict[u'organization'][u'political_level'] # noqa
try:
# index language-specific values (or it's fallback)
text_field_items = {}
for lang_code in get_langs():
search_data['title_' + lang_code] = get_localized_value(
validated_dict['title'],
lang_code
)
search_data['title_string_' + lang_code] = munge_title_to_name(
get_localized_value(validated_dict['title'], lang_code)
)
search_data['description_' + lang_code] = get_localized_value(
validated_dict['description'],
lang_code
)
search_data['keywords_' + lang_code] = get_localized_value(
validated_dict['keywords'],
lang_code
)
text_field_items['text_' + lang_code] = [get_localized_value(validated_dict['description'], lang_code)] # noqa
text_field_items['text_' + lang_code].extend(search_data['keywords_' + lang_code]) # noqa
text_field_items['text_' + lang_code].extend([r['title'][lang_code] for r in validated_dict['resources'] if r['title'][lang_code]]) # noqa
text_field_items['text_' + lang_code].extend([r['description'][lang_code] for r in validated_dict['resources'] if r['description'][lang_code]]) # noqa
# flatten values for text_* fields
for key, value in text_field_items.iteritems():
search_data[key] = ' '.join(value)
except KeyError:
pass
# log.debug(pprint.pformat(search_data))
return search_data
def before_index(self, search_data):
if not self.is_supported_package_type(search_data):
return search_data
extract_title = LangToString('title')
validated_dict = json.loads(search_data['validated_data_dict'])
search_data['res_name'] = [extract_title(r) for r in validated_dict[u'resources']] # noqa
search_data['res_description'] = [LangToString('description')(r) for r in validated_dict[u'resources']] # noqa
search_data['res_format'] = self._prepare_formats_for_index(validated_dict[u'resources']) # noqa
search_data['res_rights'] = [simplify_terms_of_use(r['rights']) for r in validated_dict[u'resources']] # noqa
search_data['title_string'] = extract_title(validated_dict)
search_data['description'] = LangToString('description')(validated_dict) # noqa
if 'political_level' in validated_dict[u'organization']:
search_data['political_level'] = validated_dict[u'organization'][u'political_level'] # noqa
try:
# index language-specific values (or it's fallback)
text_field_items = {}
for lang_code in get_langs():
search_data['title_' + lang_code] = get_localized_value(
validated_dict['title'],
lang_code
)
search_data['title_string_' + lang_code] = munge_title_to_name(
get_localized_value(validated_dict['title'], lang_code)
)
search_data['description_' + lang_code] = get_localized_value(
validated_dict['description'],
lang_code
)
search_data['keywords_' + lang_code] = get_localized_value(
validated_dict['keywords'],
lang_code
)
text_field_items['text_' + lang_code] = [get_localized_value(validated_dict['description'], lang_code)] # noqa
text_field_items['text_' + lang_code].extend(search_data['keywords_' + lang_code]) # noqa
text_field_items['text_' + lang_code].extend([r['title'][lang_code] for r in validated_dict['resources'] if r['title'][lang_code]]) # noqa
text_field_items['text_' + lang_code].extend([r['description'][lang_code] for r in validated_dict['resources'] if r['description'][lang_code]]) # noqa
# flatten values for text_* fields
for key, value in text_field_items.iteritems():
search_data[key] = ' '.join(value)
except KeyError:
pass
return search_data
def _extract_lang_value(self, value, lang_code):
new_value = sh.parse_json(value)
if isinstance(new_value, dict):
return sh.get_localized_value(new_value,
lang_code,
default_value='')
return value
def test_get_localized_value_dict(self):
lang_dict = {
'de': 'DE value',
'fr': 'FR value',
'it': 'IT value',
'en': 'EN value',
}
result = helpers.get_localized_value(lang_dict, 'de')
self.assertEquals(lang_dict['de'], result)
def test_get_localized_value_fallback(self):
lang_dict = {
'de': 'DE value',
'fr': 'FR value',
'it': 'IT value',
'en': '',
}
result = helpers.get_localized_value(lang_dict, 'en')
# if en does not exist, fallback to de
self.assertEquals(lang_dict['de'], result)
def test_get_localized_value_no_lang(self, mock_request):
mock_request.environ = {'CKAN_LANG': 'fr'}
lang_dict = {
'de': 'DE value',
'fr': 'FR value',
'it': 'IT value',
'en': 'EN value',
}
result = helpers.get_localized_value(lang_dict)
self.assertEquals(lang_dict['fr'], result)
def test_get_localized_value_invalid_dict(self):
test_dict = {"test": "dict"}
result = helpers.get_localized_value(test_dict)
self.assertEquals(test_dict, result)
def test_get_localized_value_no_lang(self, mock_request):
mock_request.environ = {"CKAN_LANG": "fr"}
lang_dict = {"de": "DE value", "fr": "FR value", "it": "IT value", "en": "EN value"}
result = helpers.get_localized_value(lang_dict)
self.assertEquals(lang_dict["fr"], result)
def test_get_localized_value_fallback(self):
lang_dict = {"de": "DE value", "fr": "FR value", "it": "IT value", "en": ""}
result = helpers.get_localized_value(lang_dict, "en")
# if en does not exist, fallback to de
self.assertEquals(lang_dict["de"], result)
def test_get_localized_value_dict(self):
lang_dict = {"de": "DE value", "fr": "FR value", "it": "IT value", "en": "EN value"}
result = helpers.get_localized_value(lang_dict, "de")
self.assertEquals(lang_dict["de"], result)
def before_index(self, search_data):
if not self.is_supported_package_type(search_data):
return search_data
extract_title = LangToString('title')
validated_dict = json.loads(search_data['validated_data_dict'])
search_data['res_name'] = [
extract_title(r) for r in validated_dict[u'resources']
] # noqa
search_data['res_name_en'] = [
sh.get_localized_value(r['title'], 'en')
for r in validated_dict[u'resources']
] # noqa
search_data['res_name_de'] = [
sh.get_localized_value(r['title'], 'de')
for r in validated_dict[u'resources']
] # noqa
search_data['res_name_fr'] = [
sh.get_localized_value(r['title'], 'fr')
for r in validated_dict[u'resources']
] # noqa
search_data['res_name_it'] = [
sh.get_localized_value(r['title'], 'it')
for r in validated_dict[u'resources']
] # noqa
search_data['res_description_en'] = [
sh.get_localized_value(r['description'], 'en')
for r in validated_dict[u'resources']
] # noqa
search_data['res_description_de'] = [
sh.get_localized_value(r['description'], 'de')
for r in validated_dict[u'resources']
] # noqa
search_data['res_description_fr'] = [
sh.get_localized_value(r['description'], 'fr')
for r in validated_dict[u'resources']
] # noqa
search_data['res_description_it'] = [
sh.get_localized_value(r['description'], 'it')
for r in validated_dict[u'resources']
] # noqa
search_data['groups_en'] = [
sh.get_localized_value(g['display_name'], 'en')
for g in validated_dict[u'groups']
] # noqa
search_data['groups_de'] = [
sh.get_localized_value(g['display_name'], 'de')
for g in validated_dict[u'groups']
] # noqa
search_data['groups_fr'] = [
sh.get_localized_value(g['display_name'], 'fr')
for g in validated_dict[u'groups']
] # noqa
search_data['groups_it'] = [
sh.get_localized_value(g['display_name'], 'it')
for g in validated_dict[u'groups']
] # noqa
search_data['res_description'] = [
LangToString('description')(r)
for r in validated_dict[u'resources']
] # noqa
search_data['res_format'] = self._prepare_formats_for_index(
validated_dict[u'resources']) # noqa
search_data['res_rights'] = [
sh.simplify_terms_of_use(r['rights'])
for r in validated_dict[u'resources']
] # noqa
search_data['title_string'] = extract_title(validated_dict)
search_data['description'] = LangToString('description')(
validated_dict) # noqa
if 'political_level' in validated_dict[u'organization']:
search_data['political_level'] = validated_dict[u'organization'][
u'political_level'] # noqa
search_data['identifier'] = validated_dict.get('identifier')
search_data['contact_points'] = [
c['name'] for c in validated_dict.get('contact_points', [])
] # noqa
search_data['publishers'] = [
p['label'] for p in validated_dict.get('publishers', [])
] # noqa
# TODO: Remove the try-except-block.
# This fixes the index while we have 'wrong' relations on
# datasets harvested with an old version of ckanext-geocat
try:
search_data['see_alsos'] = [
d['dataset_identifier']
for d in validated_dict.get('see_alsos', [])
] # noqa
except TypeError:
search_data['see_alsos'] = [
d for d in validated_dict.get('see_alsos', [])
] # noqa
# make sure we're not dealing with NoneType
if search_data['metadata_created'] is None:
search_data['metadata_created'] = ''
if search_data['metadata_modified'] is None:
search_data['metadata_modified'] = ''
try:
# index language-specific values (or it's fallback)
for lang_code in sh.get_langs():
search_data['title_' + lang_code] = sh.get_localized_value(
validated_dict['title'], lang_code)
search_data['title_string_' + lang_code] = munge_title_to_name(
sh.get_localized_value(validated_dict['title'], lang_code))
search_data['description_' +
lang_code] = sh.get_localized_value( # noqa
validated_dict['description'], lang_code)
search_data['keywords_' + lang_code] = sh.get_localized_value(
validated_dict['keywords'], lang_code)
search_data['organization_' +
lang_code] = sh.get_localized_value( # noqa
validated_dict['organization']['title'],
lang_code)
except KeyError:
pass
# clean terms for suggest context
search_data = self._prepare_suggest_context(search_data,
validated_dict)
return search_data
def _extract_lang_value(self, value, lang_code):
new_value = parse_json(value)
if isinstance(new_value, dict):
return get_localized_value(new_value, lang_code, default_value='')
return value
def test_get_localized_value_invalid_dict(self):
test_dict = {'test': 'dict'}
result = helpers.get_localized_value(test_dict)
self.assertEquals(test_dict, result)
