def vocabularies(self):
context = {'model': model, 'session': model.Session,
'user': c.user, 'for_view': True,
'with_private': False}
#c.version# = get_action('get_skos_hierarchy')(context, )
c.lastUpdate
c.currentVersion
c.lastModified
factory = ControlledVocabularyFactory()
all_mdr = factory.get_all_vocabulary_utils()
c.nals = []
for mdr, value in all_mdr.items():
name = next((lable.value_or_uri for lable in value.schema_concept_scheme.schema.prefLabel_at.values() if lable.lang == 'en'), '')
uri = value.schema_concept_scheme.uri
last_update = value.schema_concept_scheme.schema.versionInfo_owl.get('0', ResourceValue('')).value_or_uri
representation = ''
c.nals.append({
'name': name,
'uri': uri,
'lastUpdated': last_update,
'representation': ''
})
return render('configuration/vocabularies.html')
def test_mdr_factory_with_list(self):
factory = ControlledVocabularyFactory()
mdr_util = factory.get_controlled_vocabulary_util(
[factory.DATA_THEME, factory.LICENSE])
self.assertNotEqual(None, mdr_util, ' Could not instanciate mdr_util')
def test_publisher_util(self):
factory = ControlledVocabularyFactory()
publisher = factory.get_controlled_vocabulary_util(
ControlledVocabularyFactory.CORPORATE_BODY
) #type: CorporateBodiesUtil
children = publisher.get_first_level_publishers()
self.assertIsNot(None, children, "List is None")
def retrieve_all_file_types_with_context():
from odp_common.mdr.controlled_vocabulary_factory import ControlledVocabularyFactory
from odp_common.mdr.controlled_vocabulary import ControlledVocabularyUtil
factory = ControlledVocabularyFactory()
formats = factory.get_controlled_vocabulary_util(
ControlledVocabularyFactory.FILE_TYPE) # type: ControlledVocabularyUtil
list_format = formats.get_all_uris()
final_list_formats = dict.fromkeys(list_format,"format")
return final_list_formats
def controled_vocabularies_values(self, property_member):
"""
:param property_member:
:rtype: ValidationTypeResult|None
"""
try:
from odp_common.mdr.controlled_vocabulary_factory import ControlledVocabularyFactory
from odp_common.mdr.controlled_vocabulary import ControlledVocabularyUtil
object_dict = getattr(self.schema_to_validate, property_member, {})
factory = ControlledVocabularyFactory()
graphs = self.schema_to_validate.property_vocabulary_mapping.get(
property_member, '')
if not isinstance(graphs, list):
graphs = [graphs]
all_uris = []
for graph in graphs:
mdr_util = factory.get_controlled_vocabulary_util(graph)
#for uri in mdr_util.get_all_uris():
# all_uris[uri] = True
all_uris = all_uris + mdr_util.get_all_uris()
exists = True
for obj in object_dict.values(): #type: SchemaGeneric
if obj.uri not in all_uris:
exists = False
break
if exists:
return ValidationTypeResult.success
else:
log.warning(
"Validation. controled_vocabularies_values failed. Ressource [{0}] property [{1}]"
.format(self.schema_to_validate.uri, property_member))
return ValidationTypeResult.error
except BaseException as e:
log.error(
"Validation. controled_vocabularies_values failed {0}".format(
property_member))
log.debug(traceback.print_exc(e))
return None
def index(self, max_element=None):
group_type = self._guess_group_type()
context = {
'model': model,
'session': model.Session,
'user': c.user,
'for_view': True,
'with_private': False
}
if c.user:
context['with_private'] = True
locale = request.environ['CKAN_LANG'] or config.get(
'ckan.locale_default', 'en')
start = time.time()
result = get_action('get_skos_hierarchy')(context, max_element)
duration = time.time() - start
factory = ControlledVocabularyFactory()
publ_mdr = factory.get_controlled_vocabulary_util(
ControlledVocabularyFactory.CORPORATE_BODY
) #type: CorporateBodiesUtil
for top_level, item in result.items():
translation = publ_mdr.get_translation_for_language(
top_level, locale)
item['name'] = top_level.split('/')[-1].lower()
item['label'] = translation
interim = []
for child in item.get('children', []):
translation = publ_mdr.get_translation_for_language(
child[0], locale)
interim.append(
(child[0].split('/')[-1].lower(), translation, child[1]))
item['children'] = sorted(interim, key=lambda child: child[1])
log.info("get SKOS hierachy took {0}".format(duration))
c.items = result
return render(self._index_template(group_type))
def setup_template_variables(self, context, data_dict=None, package_type=None):
c = tk.c
ckan_lang = str(helpers.current_locale())
ckan_lang_fallback = str(helpers.fallback_locale())
c.is_sysadmin = new_authz.is_sysadmin(c.user)
factory = ControlledVocabularyFactory()
#self.license = factory.get_controlled_vocabulary_util(ControlledVocabularyFactory.LICENSE) #type: ControlledVocabularyUtil
# Initialize cache if needed
self.license = controlled_vocabulary_util.retrieve_all_licenses(ckan_lang)
# Initialize cache if needed
self.geographical_coverage = controlled_vocabulary_util.retrieve_all_geographical_coverage(ckan_lang)
self.controlled_keyword = controlled_vocabulary_util.retrieve_all_controlled_keyword(ckan_lang)
self.domains_eurovoc = controlled_vocabulary_util.retrieve_all_themes(ckan_lang)
self.status = controlled_vocabulary_util.retrieve_all_datasets_status(ckan_lang)
self.type_of_dataset = controlled_vocabulary_util.retrieve_all_datasets_types(ckan_lang)
self.languages = controlled_vocabulary_util.retrieve_all_languages(ckan_lang)
self.frequency = controlled_vocabulary_util.retrieve_all_frequencies(ckan_lang)
self.temporal_granularity = controlled_vocabulary_util.retrieve_all_time_periodicity(ckan_lang)
#self.notation_skos = controlled_vocabulary_util.retrieve_all_notation_skos(ckan_lang)
self.notation_skos = controlled_vocabulary_util.retrieve_all_notation_types(ckan_lang)
self.formats = factory.get_controlled_vocabulary_util(ControlledVocabularyFactory.FILE_TYPE) #type: ControlledVocabularyUtil
if (c.action in (u'edit', u'new', u'editresources', u'manage_package', u'update', 'bulk_update')):
c.license = self.license
c.geographical_coverage = self.geographical_coverage
c.controlled_keyword = self.controlled_keyword
c.domains_eurovoc = self.domains_eurovoc
c.publishers = helpers.organizations_available(c.user)
c.status = controlled_vocabulary_util.retrieve_all_datasets_status(ckan_lang)
c.type_of_dataset = controlled_vocabulary_util.retrieve_all_datasets_types(ckan_lang)
c.languages = controlled_vocabulary_util.retrieve_all_languages(ckan_lang)
c.temporal_granularity = controlled_vocabulary_util.retrieve_all_time_periodicity(ckan_lang)
c.groups = homepage.get_groups('display_name')
c.frequency = self.frequency
c.notation_skos = self.notation_skos
c.formats = self.formats.get_all_values_for_form(ckan_lang)
# datasets_titles_dict = self.retrieve_all_datasets_titles(c)
# c.datasets = datasets_titles_dict
if c.action in (u'dashboard'):
c.formats = FORMATS
# get new group name if group ID in query string
new_group_id = pylons.request.params.get('groups__0__id')
if new_group_id:
try:
data = {'id': new_group_id}
new_group = p.toolkit.get_action('group_show')(context, data)
c.new_group = new_group['name']
except p.toolkit.ObjectNotFound:
c.new_group = None
# find extras that are not part of our schema
c.additional_extras = []
schema_keys = self.create_package_schema().keys()
if c.pkg_dict:
extras = c.pkg_dict.get('extras', [])
if extras:
for extra in extras:
if not extra['key'] in schema_keys:
c.additional_extras.append(extra)
# This is messy as auths take domain object not data_dict
context_pkg = context.get('package', None)
pkg = context_pkg or c.pkg
if pkg:
try:
if not context_pkg:
context['package'] = pkg
logic.check_access('package_update', context)
c.auth_for_change_state = True
except logic.NotAuthorized:
c.auth_for_change_state = False
def index(self):
context = {'for_view': True}
start = time.time()
locale = ckanext_helpers.current_locale().language
c.package_count = ckanext_helpers.get_package_count()
c.approved_search_terms = ckanext_helpers.approved_search_terms()
c.most_viewed_datasets = ckanext_helpers.most_viewed_datasets()
c.recent_updates = ckanext_helpers.recent_updates(10)
publishers = get_action('get_skos_hierarchy')(context, None)
factory = ControlledVocabularyFactory()
publ_mdr = factory.get_controlled_vocabulary_util(ControlledVocabularyFactory.CORPORATE_BODY) #type: CorporateBodiesUtil
for top_level, item in publishers.items():
translation = publ_mdr.get_translation_for_language(top_level, locale)
item['name'] = top_level.split('/')[-1].lower()
item['label'] = translation
interim = []
for child in item.get('children', []):
translation = publ_mdr.get_translation_for_language(child[0], locale)
interim.append((child[0].split('/')[-1].lower(), translation, child[1]))
item['children'] = sorted(interim, key=lambda child: child[1])
c.get_skos_hierarchy = publishers
c.most_common_themes = get_action('theme_list')(context, {'mode': 'most_common'})
c.less_common_themes = get_action('theme_list')(context, {'mode': 'less_common'})
#c.get_eurovoc_domains_by_packages_with_cache_most =homepage.get_eurovoc_domains_by_packages_with_cache('most_common', locale)
#c.get_eurovoc_domains_by_packages_with_cache_less= homepage.get_eurovoc_domains_by_packages_with_cache('less_common', locale)
duration = time.time()-start
log.info("Build all cache took {0}".format(duration))
if c.userobj is not None:
msg = None
url = h.url_for(controller='user', action='edit')
is_google_id = \
c.userobj.name.startswith(
'https://www.google.com/accounts/o8/id')
if not c.userobj.email and (is_google_id and
not c.userobj.fullname):
msg = _(u'Please <a href="{link}">update your profile</a>'
u' and add your email address and your full name. '
u'{site} uses your email address'
u' if you need to reset your password.'.format(
link=url, site=g.site_title))
elif not c.userobj.email:
msg = _('Please <a href="%s">update your profile</a>'
' and add your email address. ') % url + \
_('%s uses your email address'
' if you need to reset your password.') \
% g.site_title
elif is_google_id and not c.userobj.fullname:
msg = _('Please <a href="%s">update your profile</a>'
' and add your full name.') % (url)
if msg:
h.flash_notice(msg, allow_html=True)
log.info("Use Home extension")
start = time.time()
response_str = base.render('home/index.html', cache_force=True)
duration = time.time() - start
log.info("Build Render index took {0}".format(duration))
# request.environ['CKAN_PAGE_CACHABLE'] = True
# response.headers["Cache-Control"] = "max-age=604800"
return response_str
# return base.render('home/static.html', cache_force=True)
def get_skos_hierarchy(context,max_element=None):
"""
:param context:
:param max_element:
:return:
"""
result = OrderedDict()
ts_query_helper = TripleStoreQueryHelpers()
user = context.get('user',None)
cache_key = ''
if user:
cache_key = 'skos_hierarchy_{0}'.format(user)
else:
cache_key = 'skos_hierarchy'
dict_string=cache.get_from_cache(cache_key, pool=cache.MISC_POOL)
if dict_string:
start_time = time.time()
result = json.loads(dict_string)
duration = time.time()-start_time
log.info("[DB] Loading json took {0}".format(duration))
else:
try:
graph_list = []
graph_list.append('dcatapop-public')
start1 = time.time()
package_count_public = ts_query_helper.get_package_count_by_publisher(graph_list)
log.info('1st package count query took {0}s'.format(time.time()-start1))
graph_list.append('dcatapop-private')
start2 = time.time()
packag_count_all = ts_query_helper.get_package_count_by_publisher(graph_list)
log.info('2nd package count query took {0}s'.format(time.time()-start2))
factory = ControlledVocabularyFactory()
publ_mdr = factory.get_controlled_vocabulary_util(ControlledVocabularyFactory.CORPORATE_BODY) #type: CorporateBodiesUtil
publ_hierarchie = publ_mdr.get_publisher_hierarchy()
for top_level, children in publ_hierarchie.items():
sum_count = 0
pub_id = top_level.split('/')[-1].lower()
if auth.has_user_permission_for_group_or_org(pub_id, user, 'read') :
sum_count += packag_count_all.get(top_level) or 0
else:
sum_count += package_count_public.get(top_level) or 0
interim = {'children': []}
for child in children:
count = 0
pub_id = child.split('/')[-1].lower()
if auth.has_user_permission_for_group_or_org(pub_id, user, 'read') :
count += packag_count_all.get(child) or 0
else:
count += package_count_public.get(child) or 0
if count > 0:
interim['children'].append((child,count))
sum_count += count
interim['total'] = sum_count
result[top_level] = interim
cache.set_value_in_cache(cache_key,json.dumps(result ), pool=cache.MISC_POOL)
except Exception, e:
log.error('Error during querying the groups in get_skos_hierarchy: %s' % e)
import traceback
log.error(traceback.print_exc())
return {}
def after_search(self, search_results, search_params):
# Translate the unselected search facets.
facets = search_results.get('search_facets')
if not facets:
return search_results
desired_lang_code = pylons.request.environ['CKAN_LANG']
fallback_lang_code = pylons.config.get('ckan.locale_default', 'en')
if desired_lang_code == fallback_lang_code:
codes = desired_lang_code
else:
codes = (desired_lang_code, fallback_lang_code)
# Look up translations for all of the facets in one db query.
overall_time = time.time()
terms = set()
for facet in facets.values():
for item in facet['items']:
terms.add(item['display_name'])
translations = ckan.logic.action.get.term_translation_show(
{'model': ckan.model}, {
'terms': terms,
'lang_codes': codes
})
factory = ControlledVocabularyFactory()
language = factory.get_controlled_vocabulary_util(
ControlledVocabularyFactory.LANGUAGE
) #type: ControlledVocabularyUtil
country = factory.get_controlled_vocabulary_util(
ControlledVocabularyFactory.COUNTRY
) #type: ControlledVocabularyUtil
eurovoc = factory.get_controlled_vocabulary_util(
ControlledVocabularyFactory.EUROVOC
) #type: ControlledVocabularyUtil
themes = factory.get_controlled_vocabulary_util(
ControlledVocabularyFactory.DATA_THEME
) #type: ControlledVocabularyUtil
format = factory.get_controlled_vocabulary_util(
ControlledVocabularyFactory.FILE_TYPE
) #type: ControlledVocabularyUtil
controlled_vocabularries = {
'vocab_theme': themes,
'res_format': format,
'vocab_concepts_eurovoc': eurovoc,
'vocab_geographical_coverage': country,
'vocab_language': language
}
for key, facet in facets.items():
if key in 'vocab_theme, res_format, vocab_concepts_eurovoc, vocab_geographical_coverage, vocab_language':
start = time.time()
for item in facet['items']:
item['display_name'] = controlled_vocabularries[
key].get_translation_for_language(
item['name'], desired_lang_code,
fallback_lang_code)
log.info('mdr {0} facet translation took {1} sec'.format(
key, (time.time() - start)))
elif key == 'vocab_catalog':
start = time.time()
for item in facet['items']:
item['display_name'] = ui_util._get_translaed_catalog(
item['name'], desired_lang_code) or item['name']
log.info('catalogue facet translation took {0} sec'.format(
time.time() - start))
elif key == 'tags':
start = time.time()
for item in facet['items']:
item['display_name'] = item['name']
log.info(
'tags facet translation took {0} sec'.format(time.time() -
start))
else:
start = time.time()
for item in facet['items']:
if item['name'] == 'true':
item['display_name'] = _('ecodp.common.private')
elif item['name'] == 'false':
item['display_name'] = _('ecodp.common.public')
matching_translations = next(
(translation for translation in translations
if translation['term'] == item['display_name']
and translation['lang_code'] == desired_lang_code),
'')
if not matching_translations:
matching_translations = next(
(translation for translation in translations
if translation['term'] == item['display_name'] and
translation['lang_code'] == fallback_lang_code),
None)
if matching_translations:
item['display_name'] = (
matching_translations['term_translation'])
log.info('rest of facets translation took {0} sec'.format(
time.time() - start))
log.info('overall facet translation took {0} sec'.format(time.time() -
overall_time))
return search_results