def before_index(self, search_data):
if not self.is_supported_package_type(search_data):
return search_data
extract_title = LangToString('title')
validated_dict = json.loads(search_data['validated_data_dict'])
# log.debug(pprint.pformat(validated_dict))
search_data['res_name'] = [extract_title(r) for r in validated_dict[u'resources']] # noqa
search_data['res_description'] = [LangToString('description')(r) for r in validated_dict[u'resources']] # noqa
search_data['res_format'] = self._prepare_formats_for_index(validated_dict[u'resources']) # noqa
search_data['res_rights'] = [simplify_terms_of_use(r['rights']) for r in validated_dict[u'resources']] # noqa
search_data['title_string'] = extract_title(validated_dict)
search_data['description'] = LangToString('description')(validated_dict) # noqa
if 'political_level' in validated_dict[u'organization']:
search_data['political_level'] = validated_dict[u'organization'][u'political_level'] # noqa
try:
# index language-specific values (or it's fallback)
text_field_items = {}
for lang_code in get_langs():
search_data['title_' + lang_code] = get_localized_value(
validated_dict['title'],
lang_code
)
search_data['title_string_' + lang_code] = munge_title_to_name(
get_localized_value(validated_dict['title'], lang_code)
)
search_data['description_' + lang_code] = get_localized_value(
validated_dict['description'],
lang_code
)
search_data['keywords_' + lang_code] = get_localized_value(
validated_dict['keywords'],
lang_code
)
text_field_items['text_' + lang_code] = [get_localized_value(validated_dict['description'], lang_code)] # noqa
text_field_items['text_' + lang_code].extend(search_data['keywords_' + lang_code]) # noqa
text_field_items['text_' + lang_code].extend([r['title'][lang_code] for r in validated_dict['resources'] if r['title'][lang_code]]) # noqa
text_field_items['text_' + lang_code].extend([r['description'][lang_code] for r in validated_dict['resources'] if r['description'][lang_code]]) # noqa
# flatten values for text_* fields
for key, value in text_field_items.iteritems():
search_data[key] = ' '.join(value)
except KeyError:
pass
# log.debug(pprint.pformat(search_data))
return search_data
def before_index(self, search_data):
if not self.is_supported_package_type(search_data):
return search_data
extract_title = LangToString('title')
validated_dict = json.loads(search_data['validated_data_dict'])
search_data['res_name'] = [extract_title(r) for r in validated_dict[u'resources']] # noqa
search_data['res_description'] = [LangToString('description')(r) for r in validated_dict[u'resources']] # noqa
search_data['res_format'] = self._prepare_formats_for_index(validated_dict[u'resources']) # noqa
search_data['res_rights'] = [simplify_terms_of_use(r['rights']) for r in validated_dict[u'resources']] # noqa
search_data['title_string'] = extract_title(validated_dict)
search_data['description'] = LangToString('description')(validated_dict) # noqa
if 'political_level' in validated_dict[u'organization']:
search_data['political_level'] = validated_dict[u'organization'][u'political_level'] # noqa
try:
# index language-specific values (or it's fallback)
text_field_items = {}
for lang_code in get_langs():
search_data['title_' + lang_code] = get_localized_value(
validated_dict['title'],
lang_code
)
search_data['title_string_' + lang_code] = munge_title_to_name(
get_localized_value(validated_dict['title'], lang_code)
)
search_data['description_' + lang_code] = get_localized_value(
validated_dict['description'],
lang_code
)
search_data['keywords_' + lang_code] = get_localized_value(
validated_dict['keywords'],
lang_code
)
text_field_items['text_' + lang_code] = [get_localized_value(validated_dict['description'], lang_code)] # noqa
text_field_items['text_' + lang_code].extend(search_data['keywords_' + lang_code]) # noqa
text_field_items['text_' + lang_code].extend([r['title'][lang_code] for r in validated_dict['resources'] if r['title'][lang_code]]) # noqa
text_field_items['text_' + lang_code].extend([r['description'][lang_code] for r in validated_dict['resources'] if r['description'][lang_code]]) # noqa
# flatten values for text_* fields
for key, value in text_field_items.iteritems():
search_data[key] = ' '.join(value)
except KeyError:
pass
return search_data
def test_simplify_terms_of_use_closed(self):
term_id = "NonCommercialNotAllowed-CommercialAllowed-ReferenceNotRequired" # noqa
result = helpers.simplify_terms_of_use(term_id)
self.assertEquals("ClosedData", result)
def test_simplify_terms_of_use_open(self):
term_id = "NonCommercialAllowed-CommercialAllowed-ReferenceRequired"
result = helpers.simplify_terms_of_use(term_id)
self.assertEquals(term_id, result)
def before_index(self, search_data):
if not self.is_supported_package_type(search_data):
return search_data
extract_title = LangToString('title')
validated_dict = json.loads(search_data['validated_data_dict'])
search_data['res_name'] = [
extract_title(r) for r in validated_dict[u'resources']
] # noqa
search_data['res_name_en'] = [
sh.get_localized_value(r['title'], 'en')
for r in validated_dict[u'resources']
] # noqa
search_data['res_name_de'] = [
sh.get_localized_value(r['title'], 'de')
for r in validated_dict[u'resources']
] # noqa
search_data['res_name_fr'] = [
sh.get_localized_value(r['title'], 'fr')
for r in validated_dict[u'resources']
] # noqa
search_data['res_name_it'] = [
sh.get_localized_value(r['title'], 'it')
for r in validated_dict[u'resources']
] # noqa
search_data['res_description_en'] = [
sh.get_localized_value(r['description'], 'en')
for r in validated_dict[u'resources']
] # noqa
search_data['res_description_de'] = [
sh.get_localized_value(r['description'], 'de')
for r in validated_dict[u'resources']
] # noqa
search_data['res_description_fr'] = [
sh.get_localized_value(r['description'], 'fr')
for r in validated_dict[u'resources']
] # noqa
search_data['res_description_it'] = [
sh.get_localized_value(r['description'], 'it')
for r in validated_dict[u'resources']
] # noqa
search_data['groups_en'] = [
sh.get_localized_value(g['display_name'], 'en')
for g in validated_dict[u'groups']
] # noqa
search_data['groups_de'] = [
sh.get_localized_value(g['display_name'], 'de')
for g in validated_dict[u'groups']
] # noqa
search_data['groups_fr'] = [
sh.get_localized_value(g['display_name'], 'fr')
for g in validated_dict[u'groups']
] # noqa
search_data['groups_it'] = [
sh.get_localized_value(g['display_name'], 'it')
for g in validated_dict[u'groups']
] # noqa
search_data['res_description'] = [
LangToString('description')(r)
for r in validated_dict[u'resources']
] # noqa
search_data['res_format'] = self._prepare_formats_for_index(
validated_dict[u'resources']) # noqa
search_data['res_rights'] = [
sh.simplify_terms_of_use(r['rights'])
for r in validated_dict[u'resources']
] # noqa
search_data['title_string'] = extract_title(validated_dict)
search_data['description'] = LangToString('description')(
validated_dict) # noqa
if 'political_level' in validated_dict[u'organization']:
search_data['political_level'] = validated_dict[u'organization'][
u'political_level'] # noqa
search_data['identifier'] = validated_dict.get('identifier')
search_data['contact_points'] = [
c['name'] for c in validated_dict.get('contact_points', [])
] # noqa
search_data['publishers'] = [
p['label'] for p in validated_dict.get('publishers', [])
] # noqa
# TODO: Remove the try-except-block.
# This fixes the index while we have 'wrong' relations on
# datasets harvested with an old version of ckanext-geocat
try:
search_data['see_alsos'] = [
d['dataset_identifier']
for d in validated_dict.get('see_alsos', [])
] # noqa
except TypeError:
search_data['see_alsos'] = [
d for d in validated_dict.get('see_alsos', [])
] # noqa
# make sure we're not dealing with NoneType
if search_data['metadata_created'] is None:
search_data['metadata_created'] = ''
if search_data['metadata_modified'] is None:
search_data['metadata_modified'] = ''
try:
# index language-specific values (or it's fallback)
for lang_code in sh.get_langs():
search_data['title_' + lang_code] = sh.get_localized_value(
validated_dict['title'], lang_code)
search_data['title_string_' + lang_code] = munge_title_to_name(
sh.get_localized_value(validated_dict['title'], lang_code))
search_data['description_' +
lang_code] = sh.get_localized_value( # noqa
validated_dict['description'], lang_code)
search_data['keywords_' + lang_code] = sh.get_localized_value(
validated_dict['keywords'], lang_code)
search_data['organization_' +
lang_code] = sh.get_localized_value( # noqa
validated_dict['organization']['title'],
lang_code)
except KeyError:
pass
# clean terms for suggest context
search_data = self._prepare_suggest_context(search_data,
validated_dict)
return search_data
def test_simplify_terms_of_use_closed(self):
term_id = 'NonCommercialNotAllowed-CommercialAllowed-ReferenceNotRequired' # noqa
result = helpers.simplify_terms_of_use(term_id)
self.assertEquals('ClosedData', result)
def test_simplify_terms_of_use_open(self):
term_id = 'NonCommercialAllowed-CommercialAllowed-ReferenceRequired'
result = helpers.simplify_terms_of_use(term_id)
self.assertEquals(term_id, result)
