def __init__(self, mdr_graph_name, mdr_class=None):
self._mdr_graph_name = mdr_graph_name
self.schema_concept_scheme = ConceptSchemeSchemaSkosWrapper(
mdr_graph_name, mdr_graph_name)
self.concept_map = {} #type dict[str,ConceptSchemaSkosWrapper]
self.schema_concept_scheme.get_description_from_ts()
self.translations = {}
self.form_values = None
self.CONCEPT_LIST_QUERY = 'select ?uri from <{0}> where {{ ?uri a <http://www.w3.org/2004/02/skos/core#Concept> . OPTIONAL {{ ?uri <http://publications.europa.eu/ontology/authority/deprecated> "false"}} .}}'
if 'true' == config.get(
'ckan.context.odp',
'false') and mdr_graph_name in FILTERED_VOCABULARIES:
self.CONCEPT_LIST_QUERY = 'select ?uri from <{0}> where {{ ?uri a <http://www.w3.org/2004/02/skos/core#Concept> . ?uri <http://lemon-model.net/lemon#context> <http://publications.europa.eu/resource/authority/use-context/ODP> . OPTIONAL {{ ?uri <http://publications.europa.eu/ontology/authority/deprecated> "false"}}.}}'
if mdr_graph_name:
query_helper = TripleStoreQueryHelpers()
query_str = self.CONCEPT_LIST_QUERY.format(mdr_graph_name)
result = query_helper.execute_select_query(query_str) or []
for res in result:
value = res.get('uri', {}).get('value')
if not CORPORATE_BODY == mdr_graph_name:
self.concept_map[value] = ConceptSchemaSkosWrapper(
value, mdr_graph_name)
else:
self.concept_map[value] = CorporateBodyWrapper(
value, mdr_graph_name)
def get_format_summary_list(self):
ts_helper = TripleStoreQueryHelpers()
raw_result = ts_helper.get_resource_formats()
common_formats = {}
for result in raw_result:
key = result['format'] or 'No format'
common_formats[key] = result['count']
return common_formats
def _openness_sores_for_dataset(self, group_id):
ts_helper = TripleStoreQueryHelpers()
ts_data =ts_helper.get_resources_of_datasets(publisher=group_id)
result = {'table': [],
'totals': ''}
totals = {'zero': 0,
'one': 0,
'two': 0,
'three': 0,
'four': 0,
'five': 0}
#input_list = (x for x in ts_data if x.get('publisher').split('/')[-1].lower() == group_id)
result_dict = {}
for dataset in ts_data:
# if not result_dict.get(dataset['dataset_name']):
# result_dict[dataset['dataset_name']] = {'dataset_name':dataset['dataset_name'],
# 'package_title': dataset['dataset_title'],
# 'publisher': dataset['publisher'],
# }
# else:
# continue
#
#
# for key,value in result_dict.iteritems():
score = model.Session.execute(
"select DISTINCT sr.entity_id, sr.value as score, ds.value as reason from task_status sr join task_status ds on sr.entity_id = ds.entity_id where sr.key = :score and ds.key = :reason and sr.entity_id = :entity",
{'entity': dataset['resource'].split('/')[-1],
'score': 'openness_score',
'reason': 'openness_score_reason'})
for row in score:
tmp = {'package_name': dataset['dataset_name'],
'package_title': dataset['dataset_title'],
'score': row[1],
'reason': row[2]}
result['table'].append(tmp)
totals_key = self._set_dataset_score(row[1])
totals[totals_key] += 1
break
result['totals'] = totals
return result
def _openness_sores_for_publisher(self):
ts_helper = TripleStoreQueryHelpers()
# score = model.Session.execute('select DISTINCT sr.entity_id, sr.value as score, ds.value as reason from task_status sr join task_status ds on sr.entity_id = ds.entity_id where sr.key = :score and ds.key = :reason',{'score':'openness_score',
# 'reason': 'openness_score_reason'})
sc_start = time.time()
query = model.Session.query(model.TaskStatus.entity_id,func.max(model.TaskStatus.value).label('value'))\
.filter(model.TaskStatus.key == u'openness_score').group_by(model.TaskStatus.entity_id)
result = query.distinct()
sc_duration = time.time()-sc_start
log.info("openness score query took {0}, {1} results".format(sc_duration, result.count()))
sc_start = time.time()
ts_data =ts_helper.get_resources_of_datasets()
sc_duration = time.time()-sc_start
log.info("ts data took {0}, {1} results".format(sc_duration, len(ts_data)))
res_dict = {}
sc_start = time.time()
for row in result:
#log.info(row)
resource = ts_data.get(row[0],None)
if resource and not res_dict.get(resource['dataset_name']):
res_dict[resource['dataset_name']] = {'ds_title': resource['dataset_title'],
'resource_list': [{'rs_id': row[0],
'value': row[1]}],
'sum_value': 0,
'owner_org': resource['publisher']}
elif resource:
res_dict[resource['dataset_name']]['resource_list'].append({'rs_id': row[0],
'value': row[1]})
sc_duration = time.time()-sc_start
log.info("openness score query 1st loop took {0}, {1} results".format(sc_duration, len(res_dict)))
sc_start = time.time()
for ds in res_dict.values():
ds['sum_value'] = str(reduce((lambda x, y: x+y), [int(rs['value']) for rs in ds['resource_list']]) / len(ds['resource_list']))
sc_duration = time.time()-sc_start
log.info("openness score query 2nd loop took {0}".format(sc_duration))
return res_dict
def test_get_all_keywords(self):
tsqh = TripleStoreQueryHelpers()
list_resources = tsqh.get_all_keywords()
self.assertTrue(list_resources)
def test_is_ckanName_unique(self):
tsqh = TripleStoreQueryHelpers()
list_resources = tsqh.is_ckanName_unique('simple-test-name-unique')
self.assertTrue(list_resources)
def test_is_ckanName_unique(self):
tsqh = TripleStoreQueryHelpers()
list_resources = tsqh.is_ckanName_unique('S2081_84_4_444_ENG')
self.assertFalse(list_resources)
def test_get_themes(self):
tsqh = TripleStoreQueryHelpers()
list_resources = tsqh.get_top_subjects()
self.assertTrue(len(list_resources)>1)
def test_get_revision_count_for_datastet(self):
tsqh = TripleStoreQueryHelpers()
list_resources = tsqh.get_revision_count_for_datastet()
self.assertTrue(len(list_resources)>1)
def test_get_revision_ids_with_issued_date(self):
tsqh = TripleStoreQueryHelpers()
list_resources = tsqh.get_revision_ids_with_issued_date()
self.assertTrue(len(list_resources)>1)
def test_get_resource_formats(self):
tsqh = TripleStoreQueryHelpers()
list_resources = tsqh.get_resource_formats()
self.assertTrue(len(list_resources)>1)
def test_get_resources_datasets(self):
tsqh = TripleStoreQueryHelpers()
list_resources = tsqh.get_resources_of_datasets()
self.assertTrue(len(list_resources)>1)
pass
def test_get_uri_datasets(self):
tsqh = TripleStoreQueryHelpers()
list_of_datasets = tsqh.get_uri_datasets()
self.assertTrue(len(list_of_datasets)>0)
def get_skos_hierarchy(context,max_element=None):
"""
:param context:
:param max_element:
:return:
"""
result = OrderedDict()
ts_query_helper = TripleStoreQueryHelpers()
user = context.get('user',None)
cache_key = ''
if user:
cache_key = 'skos_hierarchy_{0}'.format(user)
else:
cache_key = 'skos_hierarchy'
dict_string=cache.get_from_cache(cache_key, pool=cache.MISC_POOL)
if dict_string:
start_time = time.time()
result = json.loads(dict_string)
duration = time.time()-start_time
log.info("[DB] Loading json took {0}".format(duration))
else:
try:
graph_list = []
graph_list.append('dcatapop-public')
start1 = time.time()
package_count_public = ts_query_helper.get_package_count_by_publisher(graph_list)
log.info('1st package count query took {0}s'.format(time.time()-start1))
graph_list.append('dcatapop-private')
start2 = time.time()
packag_count_all = ts_query_helper.get_package_count_by_publisher(graph_list)
log.info('2nd package count query took {0}s'.format(time.time()-start2))
factory = ControlledVocabularyFactory()
publ_mdr = factory.get_controlled_vocabulary_util(ControlledVocabularyFactory.CORPORATE_BODY) #type: CorporateBodiesUtil
publ_hierarchie = publ_mdr.get_publisher_hierarchy()
for top_level, children in publ_hierarchie.items():
sum_count = 0
pub_id = top_level.split('/')[-1].lower()
if auth.has_user_permission_for_group_or_org(pub_id, user, 'read') :
sum_count += packag_count_all.get(top_level) or 0
else:
sum_count += package_count_public.get(top_level) or 0
interim = {'children': []}
for child in children:
count = 0
pub_id = child.split('/')[-1].lower()
if auth.has_user_permission_for_group_or_org(pub_id, user, 'read') :
count += packag_count_all.get(child) or 0
else:
count += package_count_public.get(child) or 0
if count > 0:
interim['children'].append((child,count))
sum_count += count
interim['total'] = sum_count
result[top_level] = interim
cache.set_value_in_cache(cache_key,json.dumps(result ), pool=cache.MISC_POOL)
except Exception, e:
log.error('Error during querying the groups in get_skos_hierarchy: %s' % e)
import traceback
log.error(traceback.print_exc())
return {}
def is_DOI_unique(name):
return TripleStoreQueryHelpers().is_DOI_unique(name)
def is_ckanName_unique(name):
return TripleStoreQueryHelpers().is_ckanName_unique(name)