def test_create_dataset_unsuccessful_404(self, mock_requests_post,
mock_fuseki_is_available):
""" Tests create gets 404 from server """
uri = "http://example.org/datasets/1"
g = Graph()
g.add((URIRef(uri), RDF.type, self.DCAT.Dataset))
mock_requests_post.return_value.status_code = 404
mock_fuseki_is_available.return_value = True
client = FusekiTriplestoreClient()
client.create_dataset_in_triplestore(g, uri)
mock_requests_post.assert_called_once_with(
'{}/data'.format(FUSEKI_ENDPOINT_URL), data=g, headers=HEADERS)
class Triplestore(CkanCommand):
'''Interacts with the triple store, e.g. reindex data.
Usage:
triplestore reindex [--dry-run]
- Reindex all datasets edited manually in the GovData portal only and which are not imported
automatically by a harvester.
'''
summary = __doc__.split('\n')[0]
usage = __doc__
def __init__(self, name):
super(Triplestore, self).__init__(name)
self.parser.add_option(
'--dry-run',
dest='dry_run',
default='True',
help='With dry-run True the reindex will be not executed. '
'The default is True.')
self.admin_user = None
self.triplestore_client = None
self.shacl_validation_client = None
self.dry_run = True
def command(self):
'''Executes commands.'''
super(Triplestore, self)._load_config()
if len(self.args) != 1:
self.parser.print_usage()
sys.exit(1)
cmd = self.args[0]
# Getting/Setting default site user
context = {
'model': model,
'session': model.Session,
'ignore_auth': True
}
self.admin_user = tk.get_action('get_site_user')(context, {})
if cmd == 'reindex':
self._check_options()
# Initialize triple store client
self.triplestore_client = FusekiTriplestoreClient()
self.shacl_validation_client = ShaclValidator()
self._reindex()
else:
print u'Command {0} not recognized'.format(cmd)
def _check_options(self):
'''Checks available options.'''
if self.options.dry_run:
if self.options.dry_run.lower() not in ('yes', 'true', 'no',
'false'):
self.parser.error('Value \'%s\' for dry-run is not a boolean!' \
% str(self.options.dry_run))
elif self.options.dry_run.lower() in ('no', 'false'):
self.dry_run = False
def _reindex(self):
'''Deletes all datasets matching package search filter query.'''
starttime = time.time()
package_obj_to_reindex = gather_dataset_ids()
endtime = time.time()
print "INFO: %s datasets found to reindex. Total time: %s." % \
(len(package_obj_to_reindex), str(endtime - starttime))
if self.dry_run:
print "INFO: DRY-RUN: The dataset reindex is disabled."
print "DEBUG: Package IDs:"
print package_obj_to_reindex.keys()
elif package_obj_to_reindex:
print 'INFO: Start updating triplestore...'
success_count = error_count = 0
starttime = time.time()
if self.triplestore_client.is_available():
for package_id, package_org in package_obj_to_reindex.iteritems(
):
try:
# Reindex package
checkpoint_start = time.time()
uri = self._update_package_in_triplestore(
package_id, package_org)
checkpoint_end = time.time()
print "DEBUG: Reindexed dataset with id %s. Time taken for reindex: %s." % \
(package_id, str(checkpoint_end - checkpoint_start))
success_count += 1
except RDFParserException as ex:
print u'ERROR: While parsing the RDF file: {0}'.format(
ex)
error_count += 1
except SPARQLWrapperException as ex:
print u'ERROR: Unexpected error while updating dataset with URI %s: %s' % (
uri, ex)
error_count += 1
except Exception as error:
print u'ERROR: While reindexing dataset with id %s. Details: %s' % \
(package_id, error.message)
error_count += 1
else:
print "INFO: TripleStore is not available. Skipping reindex!"
endtime = time.time()
print '============================================================='
print "INFO: %s datasets successfully reindexed. %s datasets couldn't reindexed. "\
"Total time: %s." % (success_count, error_count, str(endtime - starttime))
def _get_rdf(self, dataset_ref):
'''Reads the RDF presentation of the dataset with the given ID.'''
context = {'user': self.admin_user['name']}
return tk.get_action('dcat_dataset_show')(context, {'id': dataset_ref})
def _update_package_in_triplestore(self, package_id, package_org):
'''Updates the package with the given package ID in the triple store.'''
uri = 'n/a'
# Get uri of dataset
rdf = self._get_rdf(package_id)
rdf_parser = RDFParser()
rdf_parser.parse(rdf)
# Should be only one dataset
for uri in rdf_parser._datasets():
self.triplestore_client.delete_dataset_in_triplestore(uri)
self.triplestore_client.create_dataset_in_triplestore(rdf, uri)
# shacl-validate the graph
validation_rdf = self.shacl_validation_client.validate(
rdf, uri, package_org)
if validation_rdf:
# update in mqa-triplestore
self.triplestore_client.delete_dataset_in_triplestore_mqa(
uri, package_org)
self.triplestore_client.create_dataset_in_triplestore_mqa(
validation_rdf, uri)
return uri