def __init__(self, mdr_graph_name, mdr_class=None): self._mdr_graph_name = mdr_graph_name self.schema_concept_scheme = ConceptSchemeSchemaSkosWrapper( mdr_graph_name, mdr_graph_name) self.concept_map = {} #type dict[str,ConceptSchemaSkosWrapper] self.schema_concept_scheme.get_description_from_ts() self.translations = {} self.form_values = None self.CONCEPT_LIST_QUERY = 'select ?uri from <{0}> where {{ ?uri a <http://www.w3.org/2004/02/skos/core#Concept> . OPTIONAL {{ ?uri <http://publications.europa.eu/ontology/authority/deprecated> "false"}} .}}' if 'true' == config.get( 'ckan.context.odp', 'false') and mdr_graph_name in FILTERED_VOCABULARIES: self.CONCEPT_LIST_QUERY = 'select ?uri from <{0}> where {{ ?uri a <http://www.w3.org/2004/02/skos/core#Concept> . ?uri <http://lemon-model.net/lemon#context> <http://publications.europa.eu/resource/authority/use-context/ODP> . OPTIONAL {{ ?uri <http://publications.europa.eu/ontology/authority/deprecated> "false"}}.}}' if mdr_graph_name: query_helper = TripleStoreQueryHelpers() query_str = self.CONCEPT_LIST_QUERY.format(mdr_graph_name) result = query_helper.execute_select_query(query_str) or [] for res in result: value = res.get('uri', {}).get('value') if not CORPORATE_BODY == mdr_graph_name: self.concept_map[value] = ConceptSchemaSkosWrapper( value, mdr_graph_name) else: self.concept_map[value] = CorporateBodyWrapper( value, mdr_graph_name)
def get_format_summary_list(self): ts_helper = TripleStoreQueryHelpers() raw_result = ts_helper.get_resource_formats() common_formats = {} for result in raw_result: key = result['format'] or 'No format' common_formats[key] = result['count'] return common_formats
def _openness_sores_for_dataset(self, group_id): ts_helper = TripleStoreQueryHelpers() ts_data =ts_helper.get_resources_of_datasets(publisher=group_id) result = {'table': [], 'totals': ''} totals = {'zero': 0, 'one': 0, 'two': 0, 'three': 0, 'four': 0, 'five': 0} #input_list = (x for x in ts_data if x.get('publisher').split('/')[-1].lower() == group_id) result_dict = {} for dataset in ts_data: # if not result_dict.get(dataset['dataset_name']): # result_dict[dataset['dataset_name']] = {'dataset_name':dataset['dataset_name'], # 'package_title': dataset['dataset_title'], # 'publisher': dataset['publisher'], # } # else: # continue # # # for key,value in result_dict.iteritems(): score = model.Session.execute( "select DISTINCT sr.entity_id, sr.value as score, ds.value as reason from task_status sr join task_status ds on sr.entity_id = ds.entity_id where sr.key = :score and ds.key = :reason and sr.entity_id = :entity", {'entity': dataset['resource'].split('/')[-1], 'score': 'openness_score', 'reason': 'openness_score_reason'}) for row in score: tmp = {'package_name': dataset['dataset_name'], 'package_title': dataset['dataset_title'], 'score': row[1], 'reason': row[2]} result['table'].append(tmp) totals_key = self._set_dataset_score(row[1]) totals[totals_key] += 1 break result['totals'] = totals return result
def _openness_sores_for_publisher(self): ts_helper = TripleStoreQueryHelpers() # score = model.Session.execute('select DISTINCT sr.entity_id, sr.value as score, ds.value as reason from task_status sr join task_status ds on sr.entity_id = ds.entity_id where sr.key = :score and ds.key = :reason',{'score':'openness_score', # 'reason': 'openness_score_reason'}) sc_start = time.time() query = model.Session.query(model.TaskStatus.entity_id,func.max(model.TaskStatus.value).label('value'))\ .filter(model.TaskStatus.key == u'openness_score').group_by(model.TaskStatus.entity_id) result = query.distinct() sc_duration = time.time()-sc_start log.info("openness score query took {0}, {1} results".format(sc_duration, result.count())) sc_start = time.time() ts_data =ts_helper.get_resources_of_datasets() sc_duration = time.time()-sc_start log.info("ts data took {0}, {1} results".format(sc_duration, len(ts_data))) res_dict = {} sc_start = time.time() for row in result: #log.info(row) resource = ts_data.get(row[0],None) if resource and not res_dict.get(resource['dataset_name']): res_dict[resource['dataset_name']] = {'ds_title': resource['dataset_title'], 'resource_list': [{'rs_id': row[0], 'value': row[1]}], 'sum_value': 0, 'owner_org': resource['publisher']} elif resource: res_dict[resource['dataset_name']]['resource_list'].append({'rs_id': row[0], 'value': row[1]}) sc_duration = time.time()-sc_start log.info("openness score query 1st loop took {0}, {1} results".format(sc_duration, len(res_dict))) sc_start = time.time() for ds in res_dict.values(): ds['sum_value'] = str(reduce((lambda x, y: x+y), [int(rs['value']) for rs in ds['resource_list']]) / len(ds['resource_list'])) sc_duration = time.time()-sc_start log.info("openness score query 2nd loop took {0}".format(sc_duration)) return res_dict
def test_get_all_keywords(self): tsqh = TripleStoreQueryHelpers() list_resources = tsqh.get_all_keywords() self.assertTrue(list_resources)
def test_is_ckanName_unique(self): tsqh = TripleStoreQueryHelpers() list_resources = tsqh.is_ckanName_unique('simple-test-name-unique') self.assertTrue(list_resources)
def test_is_ckanName_unique(self): tsqh = TripleStoreQueryHelpers() list_resources = tsqh.is_ckanName_unique('S2081_84_4_444_ENG') self.assertFalse(list_resources)
def test_get_themes(self): tsqh = TripleStoreQueryHelpers() list_resources = tsqh.get_top_subjects() self.assertTrue(len(list_resources)>1)
def test_get_revision_count_for_datastet(self): tsqh = TripleStoreQueryHelpers() list_resources = tsqh.get_revision_count_for_datastet() self.assertTrue(len(list_resources)>1)
def test_get_revision_ids_with_issued_date(self): tsqh = TripleStoreQueryHelpers() list_resources = tsqh.get_revision_ids_with_issued_date() self.assertTrue(len(list_resources)>1)
def test_get_resource_formats(self): tsqh = TripleStoreQueryHelpers() list_resources = tsqh.get_resource_formats() self.assertTrue(len(list_resources)>1)
def test_get_resources_datasets(self): tsqh = TripleStoreQueryHelpers() list_resources = tsqh.get_resources_of_datasets() self.assertTrue(len(list_resources)>1) pass
def test_get_uri_datasets(self): tsqh = TripleStoreQueryHelpers() list_of_datasets = tsqh.get_uri_datasets() self.assertTrue(len(list_of_datasets)>0)
def get_skos_hierarchy(context,max_element=None): """ :param context: :param max_element: :return: """ result = OrderedDict() ts_query_helper = TripleStoreQueryHelpers() user = context.get('user',None) cache_key = '' if user: cache_key = 'skos_hierarchy_{0}'.format(user) else: cache_key = 'skos_hierarchy' dict_string=cache.get_from_cache(cache_key, pool=cache.MISC_POOL) if dict_string: start_time = time.time() result = json.loads(dict_string) duration = time.time()-start_time log.info("[DB] Loading json took {0}".format(duration)) else: try: graph_list = [] graph_list.append('dcatapop-public') start1 = time.time() package_count_public = ts_query_helper.get_package_count_by_publisher(graph_list) log.info('1st package count query took {0}s'.format(time.time()-start1)) graph_list.append('dcatapop-private') start2 = time.time() packag_count_all = ts_query_helper.get_package_count_by_publisher(graph_list) log.info('2nd package count query took {0}s'.format(time.time()-start2)) factory = ControlledVocabularyFactory() publ_mdr = factory.get_controlled_vocabulary_util(ControlledVocabularyFactory.CORPORATE_BODY) #type: CorporateBodiesUtil publ_hierarchie = publ_mdr.get_publisher_hierarchy() for top_level, children in publ_hierarchie.items(): sum_count = 0 pub_id = top_level.split('/')[-1].lower() if auth.has_user_permission_for_group_or_org(pub_id, user, 'read') : sum_count += packag_count_all.get(top_level) or 0 else: sum_count += package_count_public.get(top_level) or 0 interim = {'children': []} for child in children: count = 0 pub_id = child.split('/')[-1].lower() if auth.has_user_permission_for_group_or_org(pub_id, user, 'read') : count += packag_count_all.get(child) or 0 else: count += package_count_public.get(child) or 0 if count > 0: interim['children'].append((child,count)) sum_count += count interim['total'] = sum_count result[top_level] = interim cache.set_value_in_cache(cache_key,json.dumps(result ), pool=cache.MISC_POOL) except Exception, e: log.error('Error during querying the groups in get_skos_hierarchy: %s' % e) import traceback log.error(traceback.print_exc()) return {}
def is_DOI_unique(name): return TripleStoreQueryHelpers().is_DOI_unique(name)
def is_ckanName_unique(name): return TripleStoreQueryHelpers().is_ckanName_unique(name)