def vocabularies(self): context = {'model': model, 'session': model.Session, 'user': c.user, 'for_view': True, 'with_private': False} #c.version# = get_action('get_skos_hierarchy')(context, ) c.lastUpdate c.currentVersion c.lastModified factory = ControlledVocabularyFactory() all_mdr = factory.get_all_vocabulary_utils() c.nals = [] for mdr, value in all_mdr.items(): name = next((lable.value_or_uri for lable in value.schema_concept_scheme.schema.prefLabel_at.values() if lable.lang == 'en'), '') uri = value.schema_concept_scheme.uri last_update = value.schema_concept_scheme.schema.versionInfo_owl.get('0', ResourceValue('')).value_or_uri representation = '' c.nals.append({ 'name': name, 'uri': uri, 'lastUpdated': last_update, 'representation': '' }) return render('configuration/vocabularies.html')
def test_mdr_factory_with_list(self): factory = ControlledVocabularyFactory() mdr_util = factory.get_controlled_vocabulary_util( [factory.DATA_THEME, factory.LICENSE]) self.assertNotEqual(None, mdr_util, ' Could not instanciate mdr_util')
def test_publisher_util(self): factory = ControlledVocabularyFactory() publisher = factory.get_controlled_vocabulary_util( ControlledVocabularyFactory.CORPORATE_BODY ) #type: CorporateBodiesUtil children = publisher.get_first_level_publishers() self.assertIsNot(None, children, "List is None")
def retrieve_all_file_types_with_context(): from odp_common.mdr.controlled_vocabulary_factory import ControlledVocabularyFactory from odp_common.mdr.controlled_vocabulary import ControlledVocabularyUtil factory = ControlledVocabularyFactory() formats = factory.get_controlled_vocabulary_util( ControlledVocabularyFactory.FILE_TYPE) # type: ControlledVocabularyUtil list_format = formats.get_all_uris() final_list_formats = dict.fromkeys(list_format,"format") return final_list_formats
def controled_vocabularies_values(self, property_member): """ :param property_member: :rtype: ValidationTypeResult|None """ try: from odp_common.mdr.controlled_vocabulary_factory import ControlledVocabularyFactory from odp_common.mdr.controlled_vocabulary import ControlledVocabularyUtil object_dict = getattr(self.schema_to_validate, property_member, {}) factory = ControlledVocabularyFactory() graphs = self.schema_to_validate.property_vocabulary_mapping.get( property_member, '') if not isinstance(graphs, list): graphs = [graphs] all_uris = [] for graph in graphs: mdr_util = factory.get_controlled_vocabulary_util(graph) #for uri in mdr_util.get_all_uris(): # all_uris[uri] = True all_uris = all_uris + mdr_util.get_all_uris() exists = True for obj in object_dict.values(): #type: SchemaGeneric if obj.uri not in all_uris: exists = False break if exists: return ValidationTypeResult.success else: log.warning( "Validation. controled_vocabularies_values failed. Ressource [{0}] property [{1}]" .format(self.schema_to_validate.uri, property_member)) return ValidationTypeResult.error except BaseException as e: log.error( "Validation. controled_vocabularies_values failed {0}".format( property_member)) log.debug(traceback.print_exc(e)) return None
def index(self, max_element=None): group_type = self._guess_group_type() context = { 'model': model, 'session': model.Session, 'user': c.user, 'for_view': True, 'with_private': False } if c.user: context['with_private'] = True locale = request.environ['CKAN_LANG'] or config.get( 'ckan.locale_default', 'en') start = time.time() result = get_action('get_skos_hierarchy')(context, max_element) duration = time.time() - start factory = ControlledVocabularyFactory() publ_mdr = factory.get_controlled_vocabulary_util( ControlledVocabularyFactory.CORPORATE_BODY ) #type: CorporateBodiesUtil for top_level, item in result.items(): translation = publ_mdr.get_translation_for_language( top_level, locale) item['name'] = top_level.split('/')[-1].lower() item['label'] = translation interim = [] for child in item.get('children', []): translation = publ_mdr.get_translation_for_language( child[0], locale) interim.append( (child[0].split('/')[-1].lower(), translation, child[1])) item['children'] = sorted(interim, key=lambda child: child[1]) log.info("get SKOS hierachy took {0}".format(duration)) c.items = result return render(self._index_template(group_type))
def setup_template_variables(self, context, data_dict=None, package_type=None): c = tk.c ckan_lang = str(helpers.current_locale()) ckan_lang_fallback = str(helpers.fallback_locale()) c.is_sysadmin = new_authz.is_sysadmin(c.user) factory = ControlledVocabularyFactory() #self.license = factory.get_controlled_vocabulary_util(ControlledVocabularyFactory.LICENSE) #type: ControlledVocabularyUtil # Initialize cache if needed self.license = controlled_vocabulary_util.retrieve_all_licenses(ckan_lang) # Initialize cache if needed self.geographical_coverage = controlled_vocabulary_util.retrieve_all_geographical_coverage(ckan_lang) self.controlled_keyword = controlled_vocabulary_util.retrieve_all_controlled_keyword(ckan_lang) self.domains_eurovoc = controlled_vocabulary_util.retrieve_all_themes(ckan_lang) self.status = controlled_vocabulary_util.retrieve_all_datasets_status(ckan_lang) self.type_of_dataset = controlled_vocabulary_util.retrieve_all_datasets_types(ckan_lang) self.languages = controlled_vocabulary_util.retrieve_all_languages(ckan_lang) self.frequency = controlled_vocabulary_util.retrieve_all_frequencies(ckan_lang) self.temporal_granularity = controlled_vocabulary_util.retrieve_all_time_periodicity(ckan_lang) #self.notation_skos = controlled_vocabulary_util.retrieve_all_notation_skos(ckan_lang) self.notation_skos = controlled_vocabulary_util.retrieve_all_notation_types(ckan_lang) self.formats = factory.get_controlled_vocabulary_util(ControlledVocabularyFactory.FILE_TYPE) #type: ControlledVocabularyUtil if (c.action in (u'edit', u'new', u'editresources', u'manage_package', u'update', 'bulk_update')): c.license = self.license c.geographical_coverage = self.geographical_coverage c.controlled_keyword = self.controlled_keyword c.domains_eurovoc = self.domains_eurovoc c.publishers = helpers.organizations_available(c.user) c.status = controlled_vocabulary_util.retrieve_all_datasets_status(ckan_lang) c.type_of_dataset = controlled_vocabulary_util.retrieve_all_datasets_types(ckan_lang) c.languages = controlled_vocabulary_util.retrieve_all_languages(ckan_lang) c.temporal_granularity = controlled_vocabulary_util.retrieve_all_time_periodicity(ckan_lang) c.groups = homepage.get_groups('display_name') c.frequency = self.frequency c.notation_skos = self.notation_skos c.formats = self.formats.get_all_values_for_form(ckan_lang) # datasets_titles_dict = self.retrieve_all_datasets_titles(c) # c.datasets = datasets_titles_dict if c.action in (u'dashboard'): c.formats = FORMATS # get new group name if group ID in query string new_group_id = pylons.request.params.get('groups__0__id') if new_group_id: try: data = {'id': new_group_id} new_group = p.toolkit.get_action('group_show')(context, data) c.new_group = new_group['name'] except p.toolkit.ObjectNotFound: c.new_group = None # find extras that are not part of our schema c.additional_extras = [] schema_keys = self.create_package_schema().keys() if c.pkg_dict: extras = c.pkg_dict.get('extras', []) if extras: for extra in extras: if not extra['key'] in schema_keys: c.additional_extras.append(extra) # This is messy as auths take domain object not data_dict context_pkg = context.get('package', None) pkg = context_pkg or c.pkg if pkg: try: if not context_pkg: context['package'] = pkg logic.check_access('package_update', context) c.auth_for_change_state = True except logic.NotAuthorized: c.auth_for_change_state = False
def index(self): context = {'for_view': True} start = time.time() locale = ckanext_helpers.current_locale().language c.package_count = ckanext_helpers.get_package_count() c.approved_search_terms = ckanext_helpers.approved_search_terms() c.most_viewed_datasets = ckanext_helpers.most_viewed_datasets() c.recent_updates = ckanext_helpers.recent_updates(10) publishers = get_action('get_skos_hierarchy')(context, None) factory = ControlledVocabularyFactory() publ_mdr = factory.get_controlled_vocabulary_util(ControlledVocabularyFactory.CORPORATE_BODY) #type: CorporateBodiesUtil for top_level, item in publishers.items(): translation = publ_mdr.get_translation_for_language(top_level, locale) item['name'] = top_level.split('/')[-1].lower() item['label'] = translation interim = [] for child in item.get('children', []): translation = publ_mdr.get_translation_for_language(child[0], locale) interim.append((child[0].split('/')[-1].lower(), translation, child[1])) item['children'] = sorted(interim, key=lambda child: child[1]) c.get_skos_hierarchy = publishers c.most_common_themes = get_action('theme_list')(context, {'mode': 'most_common'}) c.less_common_themes = get_action('theme_list')(context, {'mode': 'less_common'}) #c.get_eurovoc_domains_by_packages_with_cache_most =homepage.get_eurovoc_domains_by_packages_with_cache('most_common', locale) #c.get_eurovoc_domains_by_packages_with_cache_less= homepage.get_eurovoc_domains_by_packages_with_cache('less_common', locale) duration = time.time()-start log.info("Build all cache took {0}".format(duration)) if c.userobj is not None: msg = None url = h.url_for(controller='user', action='edit') is_google_id = \ c.userobj.name.startswith( 'https://www.google.com/accounts/o8/id') if not c.userobj.email and (is_google_id and not c.userobj.fullname): msg = _(u'Please <a href="{link}">update your profile</a>' u' and add your email address and your full name. ' u'{site} uses your email address' u' if you need to reset your password.'.format( link=url, site=g.site_title)) elif not c.userobj.email: msg = _('Please <a href="%s">update your profile</a>' ' and add your email address. ') % url + \ _('%s uses your email address' ' if you need to reset your password.') \ % g.site_title elif is_google_id and not c.userobj.fullname: msg = _('Please <a href="%s">update your profile</a>' ' and add your full name.') % (url) if msg: h.flash_notice(msg, allow_html=True) log.info("Use Home extension") start = time.time() response_str = base.render('home/index.html', cache_force=True) duration = time.time() - start log.info("Build Render index took {0}".format(duration)) # request.environ['CKAN_PAGE_CACHABLE'] = True # response.headers["Cache-Control"] = "max-age=604800" return response_str # return base.render('home/static.html', cache_force=True)
def get_skos_hierarchy(context,max_element=None): """ :param context: :param max_element: :return: """ result = OrderedDict() ts_query_helper = TripleStoreQueryHelpers() user = context.get('user',None) cache_key = '' if user: cache_key = 'skos_hierarchy_{0}'.format(user) else: cache_key = 'skos_hierarchy' dict_string=cache.get_from_cache(cache_key, pool=cache.MISC_POOL) if dict_string: start_time = time.time() result = json.loads(dict_string) duration = time.time()-start_time log.info("[DB] Loading json took {0}".format(duration)) else: try: graph_list = [] graph_list.append('dcatapop-public') start1 = time.time() package_count_public = ts_query_helper.get_package_count_by_publisher(graph_list) log.info('1st package count query took {0}s'.format(time.time()-start1)) graph_list.append('dcatapop-private') start2 = time.time() packag_count_all = ts_query_helper.get_package_count_by_publisher(graph_list) log.info('2nd package count query took {0}s'.format(time.time()-start2)) factory = ControlledVocabularyFactory() publ_mdr = factory.get_controlled_vocabulary_util(ControlledVocabularyFactory.CORPORATE_BODY) #type: CorporateBodiesUtil publ_hierarchie = publ_mdr.get_publisher_hierarchy() for top_level, children in publ_hierarchie.items(): sum_count = 0 pub_id = top_level.split('/')[-1].lower() if auth.has_user_permission_for_group_or_org(pub_id, user, 'read') : sum_count += packag_count_all.get(top_level) or 0 else: sum_count += package_count_public.get(top_level) or 0 interim = {'children': []} for child in children: count = 0 pub_id = child.split('/')[-1].lower() if auth.has_user_permission_for_group_or_org(pub_id, user, 'read') : count += packag_count_all.get(child) or 0 else: count += package_count_public.get(child) or 0 if count > 0: interim['children'].append((child,count)) sum_count += count interim['total'] = sum_count result[top_level] = interim cache.set_value_in_cache(cache_key,json.dumps(result ), pool=cache.MISC_POOL) except Exception, e: log.error('Error during querying the groups in get_skos_hierarchy: %s' % e) import traceback log.error(traceback.print_exc()) return {}
def after_search(self, search_results, search_params): # Translate the unselected search facets. facets = search_results.get('search_facets') if not facets: return search_results desired_lang_code = pylons.request.environ['CKAN_LANG'] fallback_lang_code = pylons.config.get('ckan.locale_default', 'en') if desired_lang_code == fallback_lang_code: codes = desired_lang_code else: codes = (desired_lang_code, fallback_lang_code) # Look up translations for all of the facets in one db query. overall_time = time.time() terms = set() for facet in facets.values(): for item in facet['items']: terms.add(item['display_name']) translations = ckan.logic.action.get.term_translation_show( {'model': ckan.model}, { 'terms': terms, 'lang_codes': codes }) factory = ControlledVocabularyFactory() language = factory.get_controlled_vocabulary_util( ControlledVocabularyFactory.LANGUAGE ) #type: ControlledVocabularyUtil country = factory.get_controlled_vocabulary_util( ControlledVocabularyFactory.COUNTRY ) #type: ControlledVocabularyUtil eurovoc = factory.get_controlled_vocabulary_util( ControlledVocabularyFactory.EUROVOC ) #type: ControlledVocabularyUtil themes = factory.get_controlled_vocabulary_util( ControlledVocabularyFactory.DATA_THEME ) #type: ControlledVocabularyUtil format = factory.get_controlled_vocabulary_util( ControlledVocabularyFactory.FILE_TYPE ) #type: ControlledVocabularyUtil controlled_vocabularries = { 'vocab_theme': themes, 'res_format': format, 'vocab_concepts_eurovoc': eurovoc, 'vocab_geographical_coverage': country, 'vocab_language': language } for key, facet in facets.items(): if key in 'vocab_theme, res_format, vocab_concepts_eurovoc, vocab_geographical_coverage, vocab_language': start = time.time() for item in facet['items']: item['display_name'] = controlled_vocabularries[ key].get_translation_for_language( item['name'], desired_lang_code, fallback_lang_code) log.info('mdr {0} facet translation took {1} sec'.format( key, (time.time() - start))) elif key == 'vocab_catalog': start = time.time() for item in facet['items']: item['display_name'] = ui_util._get_translaed_catalog( item['name'], desired_lang_code) or item['name'] log.info('catalogue facet translation took {0} sec'.format( time.time() - start)) elif key == 'tags': start = time.time() for item in facet['items']: item['display_name'] = item['name'] log.info( 'tags facet translation took {0} sec'.format(time.time() - start)) else: start = time.time() for item in facet['items']: if item['name'] == 'true': item['display_name'] = _('ecodp.common.private') elif item['name'] == 'false': item['display_name'] = _('ecodp.common.public') matching_translations = next( (translation for translation in translations if translation['term'] == item['display_name'] and translation['lang_code'] == desired_lang_code), '') if not matching_translations: matching_translations = next( (translation for translation in translations if translation['term'] == item['display_name'] and translation['lang_code'] == fallback_lang_code), None) if matching_translations: item['display_name'] = ( matching_translations['term_translation']) log.info('rest of facets translation took {0} sec'.format( time.time() - start)) log.info('overall facet translation took {0} sec'.format(time.time() - overall_time)) return search_results