def update_conforms_to(pdata): cname = pdata.pop('conforms_to', None) to_delete = [] if not cname: for idx, ex in enumerate(pdata.get('extras') or []): if ex['key'] == 'conforms_to': to_delete.append(idx) cname = ex['value'] if to_delete: for idx in reversed(to_delete): pdata['extras'].pop(idx) ml_pkg = interfaces.get_for_package(pdata['id']) if ml_pkg: try: ml_conforms_to = ml_pkg['conforms_to'] except KeyError: ml_conforms_to = {} if cname: validator = toolkit.get_validator('dcatapit_conforms_to') try: # do not update conforms_to if it's already present conforms_to = json.loads(cname) if isinstance(conforms_to, list) and len(conforms_to): pdata['conforms_to'] = validator(cname, {}) return except ( ValueError, TypeError, Invalid, ), err: print(u'dataset {}: conforms_to present, but invalid: {}'.format( pdata['name'], err)).encode('utf-8') standard = {'identifier': None, 'description': {}} if not ml_conforms_to: standard['identifier'] = cname else: standard['identifier'] = ml_conforms_to.get( 'it') or ml_conforms_to.get(DEFAULT_LANG) or cname for lang, val in ml_conforms_to.items(): standard['description'][lang] = val Session.query(ML_PM).filter(ML_PM.package_id == pdata['id'], ML_PM.field == 'conforms_to').delete() pdata['conforms_to'] = json.dumps([standard])
g.add( (poc, VCARD.hasTelephone, Literal(org_dict.get('telephone')))) if 'site' in org_dict.keys(): g.add((poc, VCARD.hasURL, Literal(org_dict.get('site')))) ### Rights holder : Agent, ### holder_ref keeps graph reference to holder subject ### holder_use_dataset is a flag if holder info is taken from dataset (or organization) holder_ref, holder_use_dataset = self._add_right_holder( dataset_dict, org_dict, dataset_ref) ### Multilingual # Add localized entries in dataset # TODO: should we remove the non-localized nodes? loc_dict = interfaces.get_for_package(dataset_dict['id']) # The multilang fields loc_package_mapping = { 'title': (dataset_ref, DCT.title), 'notes': (dataset_ref, DCT.description), 'publisher_name': (publisher_ref, FOAF.name), } if holder_use_dataset and holder_ref: loc_package_mapping['holder_name'] = (holder_ref, FOAF.name) self._add_multilang_values(loc_dict, loc_package_mapping) if not holder_use_dataset and holder_ref: loc_dict = interfaces.get_for_group_or_organization(org_dict['id']) loc_package_mapping = {'name': (holder_ref, FOAF.name)} self._add_multilang_values(loc_dict, loc_package_mapping)
def test_graph_from_dataset(self): conforms_to_in = [{'identifier': 'CONF1', 'uri': 'conf01', 'title': {'en': 'title', 'it': 'title'}, 'referenceDocumentation': ['http://abc.efg/'],}, {'identifier': 'CONF2', 'title': {'en': 'title', 'it': 'title'}, 'description': {'en': 'descen', 'it': 'descit'}, 'referenceDocumentation': ['http://abc.efg/'],}, ] alternate_identifiers = [{'identifier': 'aaaabc', 'agent': {'agent_identifier': 'agent01', 'agent_name': {'en': 'Agent en 01', 'it': 'Agent it 01'}}, }, {'identifier': 'other identifier', 'agent': {}}] creators = [{'creator_name': {'en': 'abc'}, 'creator_identifier': "ABC"}, {'creator_name': {'en': 'cde'}, 'creator_identifier': "CDE"}, ] temporal_coverage = [{'temporal_start': '2001-01-01', 'temporal_end': '2001-02-01 10:11:12'}, {'temporal_start': '2001-01-01', 'temporal_end': '2001-02-01 11:12:13'}, ] subthemes = [{'theme': 'AGRI', 'subthemes': ['http://eurovoc.europa.eu/100253', 'http://eurovoc.europa.eu/100258']}, {'theme': 'ENVI', 'subthemes': []}] dataset = { 'id': '4b6fe9ca-dc77-4cec-92a4-55c6624a5bd6', 'name': 'test-dataset', 'title': 'Dataset di test DCAT_AP-IT', 'notes': 'dcatapit dataset di test', 'metadata_created': '2015-06-26T15:21:09.034694', 'metadata_modified': '2015-06-26T15:21:09.075774', 'tags': [{'name': 'Tag 1'}, {'name': 'Tag 2'}], 'issued':'2016-11-29', 'modified':'2016-11-29', 'identifier':'ISBN', 'temporal_start':'2016-11-01', 'temporal_end':'2016-11-30', 'frequency':'UPDATE_CONT', 'publisher_name':'bolzano', 'publisher_identifier':'234234234', 'creator_name':'test', 'creator_identifier':'412946129', 'holder_name':'bolzano', 'holder_identifier':'234234234', 'alternate_identifier':json.dumps(alternate_identifiers), 'temporal_coverage': json.dumps(temporal_coverage), #'theme':'ECON', 'geographical_geonames_url':'http://www.geonames.org/3181913', 'language':'{DEU,ENG,ITA}', 'is_version_of':'http://dcat.geo-solutions.it/dataset/energia-da-fonti-rinnovabili2', 'conforms_to':json.dumps(conforms_to_in), 'creator': json.dumps(creators), 'theme': json.dumps(subthemes), } pkg_id = dataset['id'] pub_names = {'it': 'IT publisher', 'es': 'EN publisher'} holder_names = {'it': 'IT holder name', 'es': 'EN holder name'} multilang_fields = [('publisher_name', 'package', k, v) for k, v in pub_names.items()] +\ [('holder_name', 'package', k, v) for k, v in holder_names.items()] pkg = helpers.call_action('package_create', {'defer_commit': True}, **dataset) rev = getattr(Session, 'revision', repo.new_revision()) Session.flush() Session.revision = rev pkg_id = pkg['id'] for field_name, field_type, lang, text in multilang_fields: interfaces.upsert_package_multilang(pkg_id, field_name, field_type, lang, text) loc_dict = interfaces.get_for_package(pkg_id) #assert loc_dict['publisher_name'] == pub_names #assert loc_dict['holder_name'] == holder_names # temporary bug for comaptibility with interfaces.get_language(), # which will return lang[0] pub_names.update({DEFAULT_LANG: dataset['publisher_name']}) # pub_names.update({DEFAULT_LANG[0]: dataset['publisher_name']}) holder_names.update({DEFAULT_LANG: dataset['holder_name']}) # holder_names.update({DEFAULT_LANG[0]: dataset['holder_name']}) s = RDFSerializer() g = s.g dataset_ref = s.graph_from_dataset(dataset) eq_(unicode(dataset_ref), utils.dataset_uri(dataset)) # Basic fields assert self._triple(g, dataset_ref, RDF.type, DCATAPIT.Dataset) assert self._triple(g, dataset_ref, DCT.title, dataset['title']) assert self._triple(g, dataset_ref, DCT.description, dataset['notes']) assert self._triple(g, dataset_ref, DCT.identifier, dataset['identifier']) # Tags eq_(len([t for t in g.triples((dataset_ref, DCAT.keyword, None))]), 2) for tag in dataset['tags']: assert self._triple(g, dataset_ref, DCAT.keyword, tag['name']) # conformsTo conforms_to = list(g.triples((None, DCT.conformsTo, None))) assert conforms_to conforms_to_dict = dict((d['identifier'], d) for d in conforms_to_in) for conf in conforms_to: conf_id = conf[-1] identifier = g.value(conf_id, DCT.identifier) titles = list(g.objects(conf_id, DCT.title)) descs = list(g.objects(conf_id, DCT.description)) references = list(g.objects(conf_id, DCATAPIT.referenceDocumentation)) check = conforms_to_dict.get(str(identifier)) assert isinstance(check, dict) if check.get('uri'): assert check['uri'] == str(conf_id) assert len(titles), "missing titles" assert (len(descs)> 0) == bool(check.get('description')), "missing descriptions" for title in titles: tlang = title.language tval = str(title) assert tval == check['title'][tlang], (tlang, tval, check['title']) for desc in descs: tlang = desc.language tval = str(desc) assert tval == check['description'][tlang], (tlang, str(tval), check['description']) ref_docs = check.get('referenceDocumentation') assert len(references) == len(ref_docs), "missing reference documentation" for dref in references: assert str(dref) in ref_docs, "{} not in {}".format(dref, ref_docs) for ref in ref_docs: assert URIRef(ref) in references # alternate identifiers alt_ids = [a[-1] for a in g.triples((None, ADMS.identifier, None))] alt_ids_dict = dict((a['identifier'], a) for a in alternate_identifiers) for alt_id in alt_ids: identifier = g.value(alt_id, SKOS.notation) check = alt_ids_dict[str(identifier)] assert str(identifier) == check['identifier'] if check.get('agent'): agent_ref = g.value(alt_id, DCT.creator) assert agent_ref is not None agent_identifier = g.value(agent_ref, DCT.identifier) agent_name = dict((v.language, str(v)) for v in g.objects(agent_ref, FOAF.name)) assert set(agent_name.items()) == set(check['agent']['agent_name'].items()),\ "expected {}, got {} for {}".format(check['agent']['agent_name'], agent_name, agent_ref) assert str(agent_identifier) == check['agent']['agent_identifier'],\ "expected {}, got {}".format(check['agent']['agent_identifier'], agent_identifier) # creators creators.append({'creator_name':{'en': 'test'}, 'creator_identifier':'412946129'}) creators_in = list(g.objects(dataset_ref, DCT.creator)) assert len(creators) == len(creators_in) for cref in creators_in: cnames = dict((str(c.language) if c.language else DEFAULT_LANG, str(c)) for c in g.objects(cref, FOAF.name)) c_identifier = g.value(cref, DCT.identifier) c_dict = {'creator_name': cnames, 'creator_identifier': str(c_identifier)} assert c_dict in creators, "no {} in {}".format(c_dict, creators) # temporal coverage temporal_coverage.append({'temporal_start': dataset['temporal_start'], 'temporal_end': dataset['temporal_end']}) temp_exts = list(g.triples((dataset_ref, DCT.temporal, None))) assert len(temp_exts) == len(temporal_coverage) # normalize values for item in temporal_coverage: for k, v in item.items(): item[k] = pdate(v) temp_ext = [] for interval_t in temp_exts: interval = interval_t[-1] start = g.value(interval, SCHEMA.startDate) end = g.value(interval, SCHEMA.endDate) assert start is not None assert end is not None temp_ext.append({'temporal_start': pdate(str(start)), 'temporal_end': pdate(str(end))}) set1 = set([tuple(d.items()) for d in temp_ext]) set2 = set([tuple(d.items()) for d in temporal_coverage]) assert set1 == set2, "Got different temporal coverage sets: \n{}\n vs\n {}".format(set1, set2) for pub_ref in g.objects(dataset_ref, DCT.publisher): _pub_names = list(g.objects(pub_ref, FOAF.name)) assert len(_pub_names) for pub_name in _pub_names: if pub_name.language: assert str(pub_name.language) in pub_names, "no {} in {}".format(pub_name.language, pub_names) assert pub_names[str(pub_name.language)] == str(pub_name), "{} vs {}".format(pub_name, pub_names) for holder_ref in g.objects(dataset_ref, DCT.rightsHolder): _holder_names = list(g.objects(holder_ref, FOAF.name)) assert len(_holder_names) for holder_name in _holder_names: if holder_name.language: assert str(holder_name.language) in holder_names, "no {} in {}".format(holder_name.language, holder_names) assert holder_names[str(holder_name.language)] == str(holder_name), "{} vs {}".format(holder_name, holder_names)
class ItalianDCATAPProfile(RDFProfile): ''' An RDF profile for the Italian DCAT-AP recommendation for data portals It requires the European DCAT-AP profile (`euro_dcat_ap`) ''' def parse_dataset(self, dataset_dict, dataset_ref): # check the dataset type if (dataset_ref, RDF.type, DCATAPIT.Dataset) not in self.g: # not a DCATAPIT dataset return dataset_dict # date info for predicate, key, logf in ( (DCT.issued, 'issued', log.debug), (DCT.modified, 'modified', log.warn), ): value = self._object_value(dataset_ref, predicate) if value: self._remove_from_extra(dataset_dict, key) value = helpers.format(value, '%Y-%m-%d', 'date') dataset_dict[key] = value else: logf('No %s found for dataset "%s"', predicate, dataset_dict.get('title', '---')) # 0..1 predicates for predicate, key, logf in ((DCT.identifier, 'identifier', log.warn), ): value = self._object_value(dataset_ref, predicate) if value: self._remove_from_extra(dataset_dict, key) dataset_dict[key] = value else: logf('No %s found for dataset "%s"', predicate, dataset_dict.get('title', '---')) # 0..n predicates list for predicate, key, logf in ( (ADMS.identifier, 'alternate_identifier', log.debug), (DCT.isVersionOf, 'is_version_of', log.debug), ): valueList = self._object_value_list(dataset_ref, predicate) if valueList: self._remove_from_extra(dataset_dict, key) value = ','.join(valueList) dataset_dict[key] = value else: logf('No %s found for dataset "%s"', predicate, dataset_dict.get('title', '---')) # conformsTo self._remove_from_extra(dataset_dict, 'conforms_to') conform_list = [] for conforms_to in self.g.objects(dataset_ref, DCT.conformsTo): conform_list.append(self._object_value(conforms_to, DCT.identifier)) if conform_list: value = ','.join(conform_list) dataset_dict['conforms_to'] = value else: log.debug('No DCT.conformsTo found for dataset "%s"', dataset_dict.get('title', '---')) # Temporal start, end = self._time_interval(dataset_ref, DCT.temporal) for v, key, logf in ( (start, 'temporal_start', log.debug), (end, 'temporal_end', log.debug), ): if v: self._remove_from_extra(dataset_dict, key) value = helpers.format(v, '%Y-%m-%d', 'date') dataset_dict[key] = value else: log.warn('No %s Date found for dataset "%s"', key, dataset_dict.get('title', '---')) # URI 0..1 for predicate, key, base_uri in ((DCT.accrualPeriodicity, 'frequency', FREQ_BASE_URI), ): valueRef = self._object_value(dataset_ref, predicate) if valueRef: self._remove_from_extra(dataset_dict, key) value = self._strip_uri(valueRef, base_uri) dataset_dict[key] = value else: log.warn('No %s found for dataset "%s"', predicate, dataset_dict.get('title', '---')) # URI lists for predicate, key, base_uri in ( (DCT.language, 'language', LANG_BASE_URI), (DCAT.theme, 'theme', THEME_BASE_URI), ): self._remove_from_extra(dataset_dict, key) valueRefList = self._object_value_list(dataset_ref, predicate) valueList = [ self._strip_uri(valueRef, base_uri) for valueRef in valueRefList ] value = ','.join(valueList) if len(valueList) > 1: value = '{' + value + '}' dataset_dict[key] = value # Spatial spatial_tags = [] geonames_url = None for spatial in self.g.objects(dataset_ref, DCT.spatial): for spatial_literal in self.g.objects( spatial, DCATAPIT.geographicalIdentifier): spatial_value = spatial_literal.value if GEO_BASE_URI in spatial_value: spatial_tags.append( self._strip_uri(spatial_value, GEO_BASE_URI)) else: if geonames_url: log.warn( "GeoName URL is already set to %s, value %s will not be imported", geonames_url, spatial_value) else: geonames_url = spatial_value if len(spatial_tags) > 0: value = ','.join(spatial_tags) if len(spatial_tags) > 1: value = '{' + value + '}' dataset_dict['geographical_name'] = value if geonames_url: dataset_dict['geographical_geonames_url'] = geonames_url ### Collect strings from multilang fields # { 'field_name': {'it': 'italian loc', 'de': 'german loc', ...}, ...} localized_dict = {} for key, predicate in ( ('title', DCT.title), ('notes', DCT.description), ): self._collect_multilang_strings(dataset_dict, key, dataset_ref, predicate, localized_dict) # Agents for predicate, basekey in ( (DCT.publisher, 'publisher'), (DCT.rightsHolder, 'holder'), (DCT.creator, 'creator'), ): agent_dict, agent_loc_dict = self._parse_agent( dataset_ref, predicate, basekey) for key, v in agent_dict.iteritems(): self._remove_from_extra(dataset_dict, key) dataset_dict[key] = v localized_dict.update(agent_loc_dict) # when all localized data have been parsed, check if there really any and add it to the dict if len(localized_dict) > 0: log.debug('Found multilang metadata') dataset_dict[LOCALISED_DICT_NAME_BASE] = localized_dict ### Resources resources_loc_dict = {} # In ckan, the license is a dataset property, not resource's # We'll collect all of the resources' licenses, then we will postprocess them licenses = [] # contains tuples (url, name) for resource_dict in dataset_dict.get('resources', []): resource_uri = resource_dict['uri'] if not resource_uri: log.warn("URI not defined for resource %s", resource_dict['name']) continue distribution = URIRef(resource_uri) if not (dataset_ref, DCAT.distribution, distribution) in self.g: log.warn("Distribution not found in dataset %s", resource_uri) continue # URI 0..1 for predicate, key, base_uri in ( (DCT['format'], 'format', FORMAT_BASE_URI), # Format ): valueRef = self._object_value(distribution, predicate) if valueRef: value = self._strip_uri(valueRef, base_uri) resource_dict[key] = value else: log.warn('No %s found for resource "%s"::"%s"', predicate, dataset_dict.get('title', '---'), resource_dict.get('name', '---')) # License license = self._object(distribution, DCT.license) if license: # just add this info in the resource extras resource_dict['license_url'] = str(license) license_name = self._object_value( license, FOAF.name) # may be either the title or the id if (license_name): # just add this info in the resource extras resource_dict['license_name'] = license_name else: license_name = "unknown" licenses.append((str(license), license_name)) else: log.warn('No license found for resource "%s"::"%s"', dataset_dict.get('title', '---'), resource_dict.get('name', '---')) # Multilang loc_dict = {} for key, predicate in ( ('name', DCT.title), ('description', DCT.description), ): self._collect_multilang_strings(resource_dict, key, distribution, predicate, loc_dict) if len(loc_dict) > 0: log.debug('Found multilang metadata in resource %s', resource_dict['name']) resources_loc_dict[resource_uri] = loc_dict if len(resources_loc_dict) > 0: log.debug('Found multilang metadata in resources') dataset_dict[LOCALISED_DICT_NAME_RESOURCES] = resources_loc_dict # postprocess licenses # license_ids = {id for url,id in licenses} # does not work in python 2.6 license_ids = set() for url, id in licenses: license_ids.add(id) if license_ids: if len(license_ids) > 1: log.warn('More than one license found for dataset "%s"', dataset_dict.get('title', '---')) dataset_dict['license_id'] = license_ids.pop() # take a random one return dataset_dict def _collect_multilang_strings(self, base_dict, key, subj, pred, loc_dict): ''' Search for multilang Literals matching (subj, pred). - Literals not localized will be stored as source_dict[key] -- possibly replacing the value set by the EURO parser - Localized literals will be stored into target_dict[key][lang] ''' for obj in self.g.objects(subj, pred): value = obj.value lang = obj.language if not lang: # force default value in dataset base_dict[key] = value else: # add localized string lang_dict = loc_dict.setdefault(key, {}) lang_dict[lang_mapping_xmllang_to_ckan.get(lang)] = value def _remove_from_extra(self, dataset_dict, key): # search and replace for extra in dataset_dict.get('extras', []): if extra['key'] == key: dataset_dict['extras'].pop(dataset_dict['extras'].index(extra)) return def _add_or_replace_extra(self, dataset_dict, key, value): # search and replace for extra in dataset_dict.get('extras', []): if extra['key'] == key: extra['value'] = value return # add if not found dataset_dict['extras'].append({'key': key, 'value': value}) def _parse_agent(self, subject, predicate, base_name): agent_dict = {} loc_dict = {} for agent in self.g.objects(subject, predicate): agent_dict[base_name + '_identifier'] = self._object_value( agent, DCT.identifier) self._collect_multilang_strings(agent_dict, base_name + '_name', agent, FOAF.name, loc_dict) return agent_dict, loc_dict def _strip_uri(self, value, base_uri): return value.replace(base_uri, '') def graph_from_dataset(self, dataset_dict, dataset_ref): title = dataset_dict.get('title') g = self.g for prefix, namespace in it_namespaces.iteritems(): g.bind(prefix, namespace) ### add a further type for the Dataset node g.add((dataset_ref, RDF.type, DCATAPIT.Dataset)) ### replace themes value = self._get_dict_value(dataset_dict, 'theme') if value: for theme in value.split(','): self.g.remove((dataset_ref, DCAT.theme, URIRef(theme))) theme = theme.replace('{', '').replace('}', '') self.g.add( (dataset_ref, DCAT.theme, URIRef(THEME_BASE_URI + theme))) self._add_concept(THEME_CONCEPTS, theme) else: self.g.add((dataset_ref, DCAT.theme, URIRef(THEME_BASE_URI + DEFAULT_THEME_KEY))) self._add_concept(THEME_CONCEPTS, DEFAULT_THEME_KEY) ### replace languages value = self._get_dict_value(dataset_dict, 'language') if value: for lang in value.split(','): self.g.remove((dataset_ref, DCT.language, Literal(lang))) lang = lang.replace('{', '').replace('}', '') self.g.add( (dataset_ref, DCT.language, URIRef(LANG_BASE_URI + lang))) # self._add_concept(LANG_CONCEPTS, lang) ### add spatial (EU URI) value = self._get_dict_value(dataset_dict, 'geographical_name') if value: for gname in value.split(','): gname = gname.replace('{', '').replace('}', '') dct_location = BNode() self.g.add((dataset_ref, DCT.spatial, dct_location)) self.g.add((dct_location, RDF['type'], DCT.Location)) # Try and add a Concept from the spatial vocabulary if self._add_concept(GEO_CONCEPTS, gname): self.g.add((dct_location, DCATAPIT.geographicalIdentifier, Literal(GEO_BASE_URI + gname))) # geo concept is not really required, but may be a useful adding self.g.add((dct_location, LOCN.geographicalName, URIRef(GEO_BASE_URI + gname))) else: # The dataset field is not a controlled tag, let's create a Concept out of the label we have concept = BNode() self.g.add((concept, RDF['type'], SKOS.Concept)) self.g.add((concept, SKOS.prefLabel, Literal(gname))) self.g.add((dct_location, LOCN.geographicalName, concept)) ### add spatial (GeoNames) value = self._get_dict_value(dataset_dict, 'geographical_geonames_url') if value: dct_location = BNode() self.g.add((dataset_ref, DCT.spatial, dct_location)) self.g.add((dct_location, RDF['type'], DCT.Location)) self.g.add((dct_location, DCATAPIT.geographicalIdentifier, Literal(value))) ### replace periodicity self._remove_node(dataset_dict, dataset_ref, ('frequency', DCT.accrualPeriodicity, None, Literal)) self._add_uri_node( dataset_dict, dataset_ref, ('frequency', DCT.accrualPeriodicity, DEFAULT_FREQ_CODE, URIRef), FREQ_BASE_URI) # self._add_concept(FREQ_CONCEPTS, dataset_dict.get('frequency', DEFAULT_VOCABULARY_KEY)) ### replace landing page self._remove_node(dataset_dict, dataset_ref, ('url', DCAT.landingPage, None, URIRef)) landing_page_uri = None if dataset_dict.get('name'): landing_page_uri = '{0}/dataset/{1}'.format( catalog_uri().rstrip('/'), dataset_dict['name']) else: landing_page_uri = dataset_uri( dataset_dict) # TODO: preserve original URI if harvested self.g.add((dataset_ref, DCAT.landingPage, URIRef(landing_page_uri))) ### conformsTo self.g.remove((dataset_ref, DCT.conformsTo, None)) value = self._get_dict_value(dataset_dict, 'conforms_to') if value: for item in value.split(','): standard = BNode() self.g.add((dataset_ref, DCT.conformsTo, standard)) self.g.add((standard, RDF['type'], DCT.Standard)) self.g.add((standard, RDF['type'], DCATAPIT.Standard)) self.g.add((standard, DCT.identifier, Literal(item))) ### publisher # DCAT by default creates this node # <dct:publisher> # <foaf:Organization rdf:about="http://10.10.100.75/organization/55535226-f82a-4cf7-903a-3e10afeaa79a"> # <foaf:name>orga2_test</foaf:name> # </foaf:Organization> # </dct:publisher> for s, p, o in g.triples((dataset_ref, DCT.publisher, None)): #log.info("Removing publisher %r", o) g.remove((s, p, o)) self._add_agent(dataset_dict, dataset_ref, 'publisher', DCT.publisher) ### Rights holder : Agent holder_ref = self._add_agent(dataset_dict, dataset_ref, 'holder', DCT.rightsHolder) ### Autore : Agent self._add_agent(dataset_dict, dataset_ref, 'creator', DCT.creator) ### Point of Contact # <dcat:contactPoint rdf:resource="http://dati.gov.it/resource/PuntoContatto/contactPointRegione_r_liguri"/> # <!-- http://dati.gov.it/resource/PuntoContatto/contactPointRegione_r_liguri --> # <dcatapit:Organization rdf:about="http://dati.gov.it/resource/PuntoContatto/contactPointRegione_r_liguri"> # <rdf:type rdf:resource="&vcard;Kind"/> # <rdf:type rdf:resource="&vcard;Organization"/> # <vcard:hasEmail rdf:resource="mailto:[email protected]"/> # <vcard:fn>Regione Liguria - Sportello Cartografico</vcard:fn> # </dcatapit:Organization> # TODO: preserve original info if harvested # retrieve the contactPoint added by the euro serializer euro_poc = g.value(subject=dataset_ref, predicate=DCAT.contactPoint, object=None, any=False) # euro poc has this format: # <dcat:contactPoint> # <vcard:Organization rdf:nodeID="Nfcd06f452bcd41f48f33c45b0c95979e"> # <vcard:fn>THE ORGANIZATION NAME</vcard:fn> # <vcard:hasEmail>THE ORGANIZATION EMAIL</vcard:hasEmail> # </vcard:Organization> # </dcat:contactPoint> if euro_poc: g.remove((dataset_ref, DCAT.contactPoint, euro_poc)) org_id = dataset_dict.get('organization', {}).get('id') # get orga info org_show = logic.get_action('organization_show') try: org_dict = org_show({}, { 'id': org_id, 'include_datasets': False, 'include_tags': False, 'include_users': False, 'include_groups': False, 'include_extras': True, 'include_followers': False }) except Exception, e: org_dict = {} org_uri = organization_uri(org_dict) poc = URIRef(org_uri) g.add((dataset_ref, DCAT.contactPoint, poc)) g.add((poc, RDF.type, DCATAPIT.Organization)) g.add((poc, RDF.type, VCARD.Kind)) g.add((poc, RDF.type, VCARD.Organization)) g.add((poc, VCARD.fn, Literal(org_dict.get('name')))) if 'email' in org_dict.keys( ): # this element is mandatory for dcatapit, but it may not have been filled for imported datasets g.add((poc, VCARD.hasEmail, URIRef(org_dict.get('email')))) if 'telephone' in org_dict.keys(): g.add( (poc, VCARD.hasTelephone, Literal(org_dict.get('telephone')))) if 'site' in org_dict.keys(): g.add((poc, VCARD.hasURL, Literal(org_dict.get('site')))) ### Multilingual # Add localized entries in dataset # TODO: should we remove the non-localized nodes? loc_dict = interfaces.get_for_package(dataset_dict['id']) # The multilang fields loc_package_mapping = { 'title': (dataset_ref, DCT.title), 'notes': (dataset_ref, DCT.description), 'holder_name': (holder_ref, FOAF.name) } self._add_multilang_values(loc_dict, loc_package_mapping) ### Resources for resource_dict in dataset_dict.get('resources', []): distribution = URIRef(resource_uri( resource_dict)) # TODO: preserve original info if harvested # Add the DCATAPIT type g.add((distribution, RDF.type, DCATAPIT.Distribution)) ### format self._remove_node(resource_dict, distribution, ('format', DCT['format'], None, Literal)) if not self._add_uri_node(resource_dict, distribution, ('distribution_format', DCT['format'], None, URIRef), FORMAT_BASE_URI): guessed_format = guess_format(resource_dict) if guessed_format: self.g.add((distribution, DCT['format'], URIRef(FORMAT_BASE_URI + guessed_format))) else: log.warn('No format for resource: %s / %s', dataset_dict.get('title', 'N/A'), resource_dict.get('description', 'N/A')) self.g.add((distribution, DCT['format'], URIRef(FORMAT_BASE_URI + DEFAULT_FORMAT_CODE))) ### license # <dct:license rdf:resource="http://creativecommons.org/licenses/by/3.0/it/"/> # # <dcatapit:LicenseDocument rdf:about="http://creativecommons.org/licenses/by/3.0/it/"> # <rdf:type rdf:resource="&dct;LicenseDocument"/> # <owl:versionInfo>3.0 ITA</owl:versionInfo> # <foaf:name>CC BY</foaf:name> # <dct:type rdf:resource="http://purl.org/adms/licencetype/Attribution"/> # </dcatapit:LicenseDocument> # "license_id" : "cc-zero" # "license_title" : "Creative Commons CCZero", # "license_url" : "http://www.opendefinition.org/licenses/cc-zero", license_url = dataset_dict.get('license_url', '') license_id = dataset_dict.get('license_id', '') license_title = dataset_dict.get('license_title', '') if license_url: license = URIRef(license_url) g.add((license, RDF['type'], DCATAPIT.LicenseDocument)) g.add((license, RDF['type'], DCT.LicenseDocument)) g.add((license, DCT['type'], URIRef('http://purl.org/adms/licencetype/Attribution') )) # TODO: infer from CKAN license g.add((distribution, DCT.license, license)) if license_id: # log.debug('Adding license id: %s', license_id) g.add((license, FOAF.name, Literal(license_id))) elif license_title: # log.debug('Adding license title: %s', license_title) g.add((license, FOAF.name, Literal(license_title))) else: g.add((license, FOAF.name, Literal('unknown'))) log.warn('License not found for dataset: %s', title) ### Multilingual # Add localized entries in resource # TODO: should we remove the not-localized nodes? loc_dict = interfaces.get_for_resource(resource_dict['id']) # The multilang fields loc_resource_mapping = { 'name': (distribution, DCT.title), 'description': (distribution, DCT.description), } self._add_multilang_values(loc_dict, loc_resource_mapping)