def before_index(self, search_data): if not self.is_supported_package_type(search_data): return search_data extract_title = LangToString('title') validated_dict = json.loads(search_data['validated_data_dict']) # log.debug(pprint.pformat(validated_dict)) search_data['res_name'] = [extract_title(r) for r in validated_dict[u'resources']] # noqa search_data['res_description'] = [LangToString('description')(r) for r in validated_dict[u'resources']] # noqa search_data['res_format'] = self._prepare_formats_for_index(validated_dict[u'resources']) # noqa search_data['res_rights'] = [simplify_terms_of_use(r['rights']) for r in validated_dict[u'resources']] # noqa search_data['title_string'] = extract_title(validated_dict) search_data['description'] = LangToString('description')(validated_dict) # noqa if 'political_level' in validated_dict[u'organization']: search_data['political_level'] = validated_dict[u'organization'][u'political_level'] # noqa try: # index language-specific values (or it's fallback) text_field_items = {} for lang_code in get_langs(): search_data['title_' + lang_code] = get_localized_value( validated_dict['title'], lang_code ) search_data['title_string_' + lang_code] = munge_title_to_name( get_localized_value(validated_dict['title'], lang_code) ) search_data['description_' + lang_code] = get_localized_value( validated_dict['description'], lang_code ) search_data['keywords_' + lang_code] = get_localized_value( validated_dict['keywords'], lang_code ) text_field_items['text_' + lang_code] = [get_localized_value(validated_dict['description'], lang_code)] # noqa text_field_items['text_' + lang_code].extend(search_data['keywords_' + lang_code]) # noqa text_field_items['text_' + lang_code].extend([r['title'][lang_code] for r in validated_dict['resources'] if r['title'][lang_code]]) # noqa text_field_items['text_' + lang_code].extend([r['description'][lang_code] for r in validated_dict['resources'] if r['description'][lang_code]]) # noqa # flatten values for text_* fields for key, value in text_field_items.iteritems(): search_data[key] = ' '.join(value) except KeyError: pass # log.debug(pprint.pformat(search_data)) return search_data
def before_index(self, search_data): if not self.is_supported_package_type(search_data): return search_data extract_title = LangToString('title') validated_dict = json.loads(search_data['validated_data_dict']) search_data['res_name'] = [extract_title(r) for r in validated_dict[u'resources']] # noqa search_data['res_description'] = [LangToString('description')(r) for r in validated_dict[u'resources']] # noqa search_data['res_format'] = self._prepare_formats_for_index(validated_dict[u'resources']) # noqa search_data['res_rights'] = [simplify_terms_of_use(r['rights']) for r in validated_dict[u'resources']] # noqa search_data['title_string'] = extract_title(validated_dict) search_data['description'] = LangToString('description')(validated_dict) # noqa if 'political_level' in validated_dict[u'organization']: search_data['political_level'] = validated_dict[u'organization'][u'political_level'] # noqa try: # index language-specific values (or it's fallback) text_field_items = {} for lang_code in get_langs(): search_data['title_' + lang_code] = get_localized_value( validated_dict['title'], lang_code ) search_data['title_string_' + lang_code] = munge_title_to_name( get_localized_value(validated_dict['title'], lang_code) ) search_data['description_' + lang_code] = get_localized_value( validated_dict['description'], lang_code ) search_data['keywords_' + lang_code] = get_localized_value( validated_dict['keywords'], lang_code ) text_field_items['text_' + lang_code] = [get_localized_value(validated_dict['description'], lang_code)] # noqa text_field_items['text_' + lang_code].extend(search_data['keywords_' + lang_code]) # noqa text_field_items['text_' + lang_code].extend([r['title'][lang_code] for r in validated_dict['resources'] if r['title'][lang_code]]) # noqa text_field_items['text_' + lang_code].extend([r['description'][lang_code] for r in validated_dict['resources'] if r['description'][lang_code]]) # noqa # flatten values for text_* fields for key, value in text_field_items.iteritems(): search_data[key] = ' '.join(value) except KeyError: pass return search_data
def test_simplify_terms_of_use_closed(self): term_id = "NonCommercialNotAllowed-CommercialAllowed-ReferenceNotRequired" # noqa result = helpers.simplify_terms_of_use(term_id) self.assertEquals("ClosedData", result)
def test_simplify_terms_of_use_open(self): term_id = "NonCommercialAllowed-CommercialAllowed-ReferenceRequired" result = helpers.simplify_terms_of_use(term_id) self.assertEquals(term_id, result)
def before_index(self, search_data): if not self.is_supported_package_type(search_data): return search_data extract_title = LangToString('title') validated_dict = json.loads(search_data['validated_data_dict']) search_data['res_name'] = [ extract_title(r) for r in validated_dict[u'resources'] ] # noqa search_data['res_name_en'] = [ sh.get_localized_value(r['title'], 'en') for r in validated_dict[u'resources'] ] # noqa search_data['res_name_de'] = [ sh.get_localized_value(r['title'], 'de') for r in validated_dict[u'resources'] ] # noqa search_data['res_name_fr'] = [ sh.get_localized_value(r['title'], 'fr') for r in validated_dict[u'resources'] ] # noqa search_data['res_name_it'] = [ sh.get_localized_value(r['title'], 'it') for r in validated_dict[u'resources'] ] # noqa search_data['res_description_en'] = [ sh.get_localized_value(r['description'], 'en') for r in validated_dict[u'resources'] ] # noqa search_data['res_description_de'] = [ sh.get_localized_value(r['description'], 'de') for r in validated_dict[u'resources'] ] # noqa search_data['res_description_fr'] = [ sh.get_localized_value(r['description'], 'fr') for r in validated_dict[u'resources'] ] # noqa search_data['res_description_it'] = [ sh.get_localized_value(r['description'], 'it') for r in validated_dict[u'resources'] ] # noqa search_data['groups_en'] = [ sh.get_localized_value(g['display_name'], 'en') for g in validated_dict[u'groups'] ] # noqa search_data['groups_de'] = [ sh.get_localized_value(g['display_name'], 'de') for g in validated_dict[u'groups'] ] # noqa search_data['groups_fr'] = [ sh.get_localized_value(g['display_name'], 'fr') for g in validated_dict[u'groups'] ] # noqa search_data['groups_it'] = [ sh.get_localized_value(g['display_name'], 'it') for g in validated_dict[u'groups'] ] # noqa search_data['res_description'] = [ LangToString('description')(r) for r in validated_dict[u'resources'] ] # noqa search_data['res_format'] = self._prepare_formats_for_index( validated_dict[u'resources']) # noqa search_data['res_rights'] = [ sh.simplify_terms_of_use(r['rights']) for r in validated_dict[u'resources'] ] # noqa search_data['title_string'] = extract_title(validated_dict) search_data['description'] = LangToString('description')( validated_dict) # noqa if 'political_level' in validated_dict[u'organization']: search_data['political_level'] = validated_dict[u'organization'][ u'political_level'] # noqa search_data['identifier'] = validated_dict.get('identifier') search_data['contact_points'] = [ c['name'] for c in validated_dict.get('contact_points', []) ] # noqa search_data['publishers'] = [ p['label'] for p in validated_dict.get('publishers', []) ] # noqa # TODO: Remove the try-except-block. # This fixes the index while we have 'wrong' relations on # datasets harvested with an old version of ckanext-geocat try: search_data['see_alsos'] = [ d['dataset_identifier'] for d in validated_dict.get('see_alsos', []) ] # noqa except TypeError: search_data['see_alsos'] = [ d for d in validated_dict.get('see_alsos', []) ] # noqa # make sure we're not dealing with NoneType if search_data['metadata_created'] is None: search_data['metadata_created'] = '' if search_data['metadata_modified'] is None: search_data['metadata_modified'] = '' try: # index language-specific values (or it's fallback) for lang_code in sh.get_langs(): search_data['title_' + lang_code] = sh.get_localized_value( validated_dict['title'], lang_code) search_data['title_string_' + lang_code] = munge_title_to_name( sh.get_localized_value(validated_dict['title'], lang_code)) search_data['description_' + lang_code] = sh.get_localized_value( # noqa validated_dict['description'], lang_code) search_data['keywords_' + lang_code] = sh.get_localized_value( validated_dict['keywords'], lang_code) search_data['organization_' + lang_code] = sh.get_localized_value( # noqa validated_dict['organization']['title'], lang_code) except KeyError: pass # clean terms for suggest context search_data = self._prepare_suggest_context(search_data, validated_dict) return search_data
def test_simplify_terms_of_use_closed(self): term_id = 'NonCommercialNotAllowed-CommercialAllowed-ReferenceNotRequired' # noqa result = helpers.simplify_terms_of_use(term_id) self.assertEquals('ClosedData', result)
def test_simplify_terms_of_use_open(self): term_id = 'NonCommercialAllowed-CommercialAllowed-ReferenceRequired' result = helpers.simplify_terms_of_use(term_id) self.assertEquals(term_id, result)