def test_select_datasets_in_triplestore_harvest_info( self, mock_sparql_query, mock_sparql_set_query, mock_fuseki_is_available): """ Test if query for selection is set properly and if results returned """ owner_org = 'org-1' mock_response = {"foo": "bar"} test_query = GET_URIS_FROM_HARVEST_INFO_QUERY % { 'owner_org': owner_org } mock_fuseki_is_available.return_value = True mock_sparql_query.return_value = mock_response client = FusekiTriplestoreClient() result = client.select_datasets_in_triplestore_harvest_info(test_query) self.assertEquals(result, mock_response) mock_sparql_query.assert_called_once_with() mock_sparql_set_query.assert_called_with(test_query)
class DCATdeRDFHarvester(DCATRDFHarvester): """ DCAT-AP.de RDF Harvester """ p.implements(IDCATRDFHarvester) # -- begin IDCATRDFHarvester implementation -- # pylint: disable=C0111,W0613,R0201,C0103 def before_download(self, url, harvest_job): return url, [] def update_session(self, session): return session def after_download(self, content, harvest_job): return content, [] def after_parsing(self, rdf_parser, harvest_job): """ Insert harvested data into triplestore and validate the data """ error_messages = [] if rdf_parser and self.triplestore_client.is_available(): LOGGER.debug(u'Start updating triplestore...') source_dataset = model.Package.get(harvest_job.source.id) org_available = True if not source_dataset or not hasattr(source_dataset, 'owner_org'): org_available = False LOGGER.warn(u'There is no organization specified in the harvest source. SHACL validation ' \ u'will be deactivated!') for uri in rdf_parser._datasets(): LOGGER.debug(u'Process URI: %s', uri) try: self.triplestore_client.delete_dataset_in_triplestore(uri) triples = rdf_parser.g.query( GET_DATASET_BY_URI_SPARQL_QUERY % {'uri': uri}) if triples: graph = Graph() for triple in triples: graph.add(triple) rdf_graph = graph.serialize(format="xml") self.triplestore_client.create_dataset_in_triplestore( rdf_graph, uri) if org_available: # save harvesting info harvest_graph = Graph() harvest_graph.bind("foaf", FOAF) harvest_graph.add( (URIRef(uri), FOAF.knows, Literal(source_dataset.owner_org))) rdf_harvest_graph = harvest_graph.serialize( format="xml") self.triplestore_client.create_dataset_in_triplestore_harvest_info( rdf_harvest_graph, uri) if org_available: # SHACL Validation validation_errors = self._validate_dataset_rdf_graph( uri, rdf_graph, source_dataset) error_messages.extend(validation_errors) else: LOGGER.warn( u'Could not find triples to URI %s. Updating is not possible.', uri) except SPARQLWrapperException as exception: LOGGER.error( u'Unexpected error while deleting dataset with URI %s in TripleStore: %s', uri, exception) error_messages.append( u'Error while deleting dataset from TripleStore: %s' % exception) LOGGER.debug(u'Finished updating triplestore.') return rdf_parser, error_messages def before_update(self, harvest_object, dataset_dict, temp_dict): pass def after_update(self, harvest_object, dataset_dict, temp_dict): return None def before_create(self, harvest_object, dataset_dict, temp_dict): pass def after_create(self, harvest_object, dataset_dict, temp_dict): return None def update_package_schema_for_create(self, package_schema): try: package_schema['maintainer_email'].remove(self.email_validator) package_schema['author_email'].remove(self.email_validator) except ValueError: pass return package_schema def update_package_schema_for_update(self, package_schema): try: package_schema['maintainer_email'].remove(self.email_validator) package_schema['author_email'].remove(self.email_validator) except ValueError: pass return package_schema # pylint: enable=C0111,W0613,R0201,C0103 # -- end IDCATRDFHarvester implementation -- def __init__(self, name='dcatde_rdf'): ''' Set global parameters from config ''' DCATRDFHarvester.__init__(self) self.triplestore_client = FusekiTriplestoreClient() self.shacl_validator_client = ShaclValidator() self.licenses_upgrade = {} license_file = pylons.config.get( 'ckanext.dcatde.urls.dcat_licenses_upgrade_mapping') if license_file: self.licenses_upgrade = load_json_mapping( license_file, "DCAT License upgrade mapping", LOGGER) try: self.email_validator = toolkit.get_validator('email_validator') except UnknownValidator: pass def info(self): return { 'name': 'dcatde_rdf', 'title': 'DCAT-AP.de RDF Harvester', 'description': 'Harvester for DCAT-AP.de datasets from an RDF graph' } @staticmethod def _get_portal_from_config(source_config): if source_config: return json.loads(source_config).get(CONFIG_PARAM_HARVESTED_PORTAL) return '' @staticmethod def _get_resources_required_config(source_config): if source_config: return json.loads(source_config).get( CONFIG_PARAM_RESOURCES_REQUIRED, False) return False @staticmethod def _get_fallback_license(): fallback = pylons.config.get( 'ckanext.dcatde.harvest.default_license', 'http://dcat-ap.de/def/licenses/other-closed') return fallback def _mark_datasets_for_deletion(self, guids_in_source, harvest_job): # This is the same as the method in the base class, except that a different query is used. object_ids = [] portal = self._get_portal_from_config(harvest_job.source.config) starttime = time.time() # Get all previous current guids and dataset ids for this harvested portal independent of # the harvest objects. This allows cleaning the harvest data without loosing the # dataset mappings. # Build a subquery to get all active packages having a GUID first # pylint: disable=E1101 subquery = model.Session.query(model.PackageExtra.value, model.Package.id) \ .join(model.Package, model.Package.id == model.PackageExtra.package_id)\ .filter(model.Package.state == model.State.ACTIVE) \ .filter(model.PackageExtra.state == model.State.ACTIVE) \ .filter(model.PackageExtra.key == 'guid') \ .subquery() # then get all active packages of the current portal and join with their GUIDs if # available (outer join) query = model.Session.query(model.Package.id, subquery.c.value) \ .join(model.PackageExtra, model.PackageExtra.package_id == model.Package.id)\ .outerjoin(subquery, subquery.c.id == model.Package.id)\ .filter(model.Package.state == model.State.ACTIVE) \ .filter(model.PackageExtra.state == model.State.ACTIVE) \ .filter(model.PackageExtra.key == EXTRA_KEY_HARVESTED_PORTAL) \ .filter(model.PackageExtra.value == portal) # pylint: enable=E1101 checkpoint_start = time.time() guid_to_package_id = {} for package_id, guid in query: if guid: guid_to_package_id[guid] = package_id # Also remove all packages without a GUID, use ID as GUID to share logic below else: guid_to_package_id[package_id] = package_id checkpoint_end = time.time() LOGGER.debug('Time for query harvest source related datasets : %s', str(checkpoint_end - checkpoint_start)) guids_in_db = guid_to_package_id.keys() # Get objects/datasets to delete (ie in the DB but not in the source) guids_in_source_unique = set(guids_in_source) guids_in_db_unique = set(guids_in_db) LOGGER.debug('guids in source: %s, unique guids in source: %s, '\ 'guids in db: %s, unique guids in db: %s', len(guids_in_source), len(guids_in_source_unique), len(guids_in_db), len(guids_in_db_unique)) guids_to_delete = guids_in_db_unique - guids_in_source_unique # Create a harvest object for each of them, flagged for deletion for guid in guids_to_delete: obj = HarvestObject( guid=guid, job=harvest_job, package_id=guid_to_package_id[guid], extras=[HarvestObjectExtra(key='status', value='delete')]) # Mark the rest of objects for this guid as not current model.Session.query(HarvestObject) \ .filter_by(guid=guid) \ .update({'current': False}, False) obj.save() object_ids.append(obj.id) endtime = time.time() LOGGER.debug('Found %s packages for deletion. Time total: %s', len(guids_to_delete), str(endtime - starttime)) self._delete_deprecated_datasets_from_triplestore( guids_in_source_unique, guids_to_delete, harvest_job) return object_ids def _amend_package(self, harvest_object): ''' Amend package information. ''' package = json.loads(harvest_object.content) if 'extras' not in package: package['extras'] = [] portal = self._get_portal_from_config(harvest_object.source.config) set_extras_field(package, EXTRA_KEY_HARVESTED_PORTAL, portal) # ensure all resources have a (recent) license for resource in package.get('resources', []): log_prefix = u'{0}: Resource {1} of package {2} (GUID {3})'.format( harvest_object.source.title, resource.get('uri', ''), package.get('name', ''), harvest_object.guid) if resource.get(RES_EXTRA_KEY_LICENSE, '') == '': LOGGER.info(log_prefix + u' has no license. Adding default value.') resource[RES_EXTRA_KEY_LICENSE] = self._get_fallback_license() elif self.licenses_upgrade: current_license = resource.get(RES_EXTRA_KEY_LICENSE) new_license = self.licenses_upgrade.get(current_license, '') if new_license == '': LOGGER.info(log_prefix + u' has a deprecated or unknown license {0}. '\ u'Keeping old value.'.format(current_license)) elif current_license != new_license: LOGGER.info(log_prefix + u' had old license {0}. '\ u'Updated value to recent DCAT list.'.format(current_license)) resource[RES_EXTRA_KEY_LICENSE] = new_license # write changes back to harvest object content harvest_object.content = json.dumps(package) def _skip_datasets_without_resource(self, harvest_object): ''' Checks if resources are present when configured. ''' package = json.loads(harvest_object.content) if (self._get_resources_required_config(harvest_object.source.config)\ and not package.get('resources')): # write details to log LOGGER.info( u'%s: Resources are required, but dataset %s (GUID %s) has none. Skipping dataset.', harvest_object.source.title, package.get('name', ''), harvest_object.guid) return True return False def import_stage(self, harvest_object): ''' Import stage for the DCAT-AP.de harvester. ''' LOGGER.debug('In DCATdeRDFHarvester import_stage') # override delete logic status = self._get_object_extra(harvest_object, 'status') if status == 'delete': if not harvest_object.package_id: LOGGER.warn( u'Harvest object with status delete contains no package id for guid %s', harvest_object.guid) return False HarvestUtils.rename_delete_dataset_with_id( harvest_object.package_id) self._delete_dataset_in_triplestore(harvest_object) LOGGER.info(u'Deleted package %s with guid %s', harvest_object.package_id, harvest_object.guid) return True # skip if resources are not present when configured if self._skip_datasets_without_resource(harvest_object): info_deleted_local_dataset = '' datasets_from_db = self._read_datasets_from_db(harvest_object.guid) if datasets_from_db: if len(datasets_from_db) == 1: HarvestUtils.rename_delete_dataset_with_id( datasets_from_db[0][0]) LOGGER.info(u'Deleted local dataset with GUID %s as harvest object has '\ u'no resources.', harvest_object.guid) info_deleted_local_dataset = ' Local dataset without resources deleted.' else: LOGGER.warn( u'Not deleting package with GUID %s, because more than one dataset was found!', harvest_object.guid) info_deleted_local_dataset = ' More than one local dataset with the same GUID!' # do not include details in error such that they get summarized in the UI self._save_object_error( 'Dataset has no resources, but they are required by config. Skipping.{0}' .format(info_deleted_local_dataset), harvest_object, 'Import') return False # set custom field and perform other fixes on the data self._amend_package(harvest_object) import_dataset = HarvestUtils.handle_duplicates(harvest_object.content) if import_dataset: return super(DCATdeRDFHarvester, self).import_stage(harvest_object) self._save_object_error( 'Skipping importing dataset, because of duplicate detection!', harvest_object, 'Import') return False def validate_config(self, source_config): ''' Validates additional configuration parameters for DCAT-AP.de harvester. ''' cfg = super(DCATdeRDFHarvester, self).validate_config(source_config) if cfg: config_obj = json.loads(cfg) if CONFIG_PARAM_HARVESTED_PORTAL in config_obj: harvested_portal = config_obj[CONFIG_PARAM_HARVESTED_PORTAL] if not isinstance(harvested_portal, basestring): raise ValueError('%s must be a string' % CONFIG_PARAM_HARVESTED_PORTAL) else: raise KeyError('%s is not set in config.' % CONFIG_PARAM_HARVESTED_PORTAL) else: raise ValueError('The parameter %s has to be set in the configuration.' % \ CONFIG_PARAM_HARVESTED_PORTAL) return cfg def _delete_dataset_in_triplestore(self, harvest_object): ''' Deletes the package with the given package ID in the triple store. ''' try: if self.triplestore_client.is_available(): package_id = harvest_object.package_id LOGGER.debug( u'Start deleting dataset with ID %s from triplestore.', package_id) context = {'user': self._get_user_name()} rdf = toolkit.get_action('dcat_dataset_show')(context, { 'id': package_id }) rdf_parser = RDFParser() rdf_parser.parse(rdf) # Should be only one dataset uri = next(rdf_parser._datasets(), None) source_dataset = model.Package.get(harvest_object.source.id) self._delete_dataset_in_triplestore_by_uri(uri, source_dataset) except RDFParserException as ex: LOGGER.warn( u'Error while parsing the RDF file for dataset with ID %s: %s', package_id, ex) def _delete_dataset_in_triplestore_by_uri(self, uri, source_dataset): ''' Deletes the package with the given URI in the triple store. ''' try: if self.triplestore_client.is_available(): LOGGER.debug( u'Start deleting dataset with URI %s from triplestore.', uri) if uri: self.triplestore_client.delete_dataset_in_triplestore(uri) if source_dataset and hasattr(source_dataset, 'owner_org'): self.triplestore_client.delete_dataset_in_triplestore_mqa( uri, source_dataset.owner_org) self.triplestore_client.delete_dataset_in_triplestore_harvest_info( uri, source_dataset.owner_org) LOGGER.debug( u'Successfully deleted dataset with URI %s from triplestore.', uri) else: LOGGER.debug(u'URI could not determined. Skip deleting.') except SPARQLWrapperException as ex: LOGGER.warn( u'Error while deleting dataset with URI %s from triplestore: %s', uri, ex) def _validate_dataset_rdf_graph(self, uri, rdf_graph, source_dataset): ''' Validates the package rdf graph with the given URI and saves the validation report in the triple store. ''' error_messages = [] try: result = self.shacl_validator_client.validate( rdf_graph, uri, source_dataset.owner_org) self.triplestore_client.delete_dataset_in_triplestore_mqa( uri, source_dataset.owner_org) self.triplestore_client.create_dataset_in_triplestore_mqa( result, uri) except SPARQLWrapperException as exception: LOGGER.error(u'Unexpected error while deleting SHACL validation report for ' \ u'dataset with URI %s: %s', uri, exception) error_messages.append(u'Error while deleting SHACL report: %s' % exception) return error_messages def _delete_deprecated_datasets_from_triplestore(self, harvested_uris, uris_db_marked_deleted, harvest_job): ''' Check for deprecated datasets (not stored in CKAN) in the triplestore and delete them from all datastores. ''' # get owner org source_dataset = model.Package.get(harvest_job.source.id) if source_dataset and hasattr(source_dataset, 'owner_org'): # Read URIs from harvest_info datastore existing_uris = self._get_existing_dataset_uris_from_triplestore( source_dataset.owner_org) # compare existing with harvested URIs to see which URIs were not updated existing_uris_unique = set(existing_uris) harvested_uris_unique = set(harvested_uris) uris_to_be_deleted = (existing_uris_unique - harvested_uris_unique ) - set(uris_db_marked_deleted) LOGGER.info( u'Found %s harvesting URIs in the triplestore that are no longer provided.', len(uris_to_be_deleted)) # delete deprecated datasets from triplestore for dataset_uri in uris_to_be_deleted: LOGGER.info(u'Delete <%s> from all triplestore datastores.', dataset_uri) self._delete_dataset_in_triplestore_by_uri( dataset_uri, source_dataset) else: LOGGER.info(u'Harvest source %s, harvest job %s: "owner_org" NOT found. Cannot retrieve the ' \ u'harvested URIs to the harvest source. Deprecated datasets which are not ' \ u'stored in CKAN will not be deleted properly from the triplestore.', harvest_job.source.id, harvest_job.id) def _get_existing_dataset_uris_from_triplestore(self, owner_org): ''' Requests all URIs from the harvest_info datastore and returns them as a list. ''' existing_uris = [] query = GET_URIS_FROM_HARVEST_INFO_QUERY % {'owner_org': owner_org} raw_response = self.triplestore_client.select_datasets_in_triplestore_harvest_info( query) response = raw_response.convert() for res in response["results"]["bindings"]: if "s" in res: existing_uris.append(res["s"]["value"]) return existing_uris