def test_get_static_enumeration(self):
rq, _ = self.loader.getTextForName('test-enum')
metadata = gquery.get_metadata(rq, '')
self.assertIn('enumerate', metadata, 'Should contain enumerate')
enumeration = gquery.get_enumeration(rq, 'o',
'http://mock-endpoint/sparql',
metadata)
self.assertIsInstance(enumeration, list,
'Should return a list of values')
self.assertEquals(len(enumeration), 2, 'Should have two elements')
def test_get_metadata(self):
rq, _ = self.loader.getTextForName('test-sparql')
metadata = gquery.get_metadata(rq, '')
self.assertIn('type', metadata, 'Should have a type field')
self.assertIn('variables', metadata, 'Should have a variables field')
self.assertEqual(metadata['type'], 'SelectQuery',
'Should be type SelectQuery')
self.assertIsInstance(
metadata['variables'], list, 'Should be a list of variables')
for var in metadata['variables']:
self.assertIsInstance(var, rdflib.term.Variable,
'Should be of type Variable')
def dispatchSPARQLQuery(raw_sparql_query, loader, requestArgs, acceptHeader,
content, formData, requestUrl):
"""Executes the specified SPARQL query."""
endpoint, auth = gquery.guess_endpoint_uri(raw_sparql_query, loader)
if endpoint == '':
return 'No SPARQL endpoint indicated', 407, {}
glogger.debug("=====================================================")
glogger.debug("Sending query to SPARQL endpoint: {}".format(endpoint))
glogger.debug("=====================================================")
try:
query_metadata = gquery.get_metadata(raw_sparql_query, endpoint)
except Exception as e:
# extracting metadata
return {'error': str(e)}, 400, {}
acceptHeader = 'application/json' if isinstance(raw_sparql_query,
dict) else acceptHeader
pagination = query_metadata[
'pagination'] if 'pagination' in query_metadata else ""
rewritten_query = query_metadata['query']
# Rewrite query using parameter values
if query_metadata['type'] == 'SelectQuery' or query_metadata[
'type'] == 'ConstructQuery':
rewritten_query = gquery.rewrite_query(
query_metadata['original_query'], query_metadata['parameters'],
requestArgs)
# Rewrite query using pagination
if query_metadata[
'type'] == 'SelectQuery' and 'pagination' in query_metadata:
rewritten_query = gquery.paginate_query(rewritten_query,
query_metadata['pagination'],
requestArgs)
resp = None
headers = {}
# If we have a mime field, we load the remote dump and query it locally
if 'mime' in query_metadata and query_metadata['mime']:
glogger.debug(
"Detected {} MIME type, proceeding with locally loading remote dump"
.format(query_metadata['mime']))
g = Graph()
try:
query_metadata = gquery.get_metadata(raw_sparql_query, endpoint)
g.parse(endpoint, format=query_metadata['mime'])
glogger.debug(
"Local RDF graph loaded successfully with {} triples".format(
len(g)))
except Exception as e:
glogger.error(e)
results = g.query(rewritten_query, result='sparql')
# Prepare return format as requested
resp_string = ""
if 'application/json' in acceptHeader or (
content and 'application/json' in static.mimetypes[content]):
resp_string = results.serialize(format='json')
glogger.debug(
"Results of SPARQL query against locally loaded dump: {}".
format(resp_string))
elif 'text/csv' in acceptHeader or (content and 'text/csv'
in static.mimetypes[content]):
resp_string = results.serialize(format='csv')
glogger.debug(
"Results of SPARQL query against locally loaded dump: {}".
format(resp_string))
else:
return 'Unacceptable requested format', 415, {}
glogger.debug(
"Finished processing query against RDF dump, end of use case")
del g
# Check for INSERT/POST
elif query_metadata['type'] == 'InsertData':
glogger.debug("Processing INSERT query")
# Rewrite INSERT
rewritten_query = rewritten_query.replace(
"?_g_iri", "{}".format(formData.get('g')))
rewritten_query = rewritten_query.replace("<s> <p> <o>",
formData.get('data'))
glogger.debug("INSERT query rewritten as {}".format(rewritten_query))
# Prepare HTTP POST request
reqHeaders = {
'Accept': acceptHeader,
'Content-Type': 'application/sparql-update'
}
response = requests.post(endpoint,
data=rewritten_query,
headers=reqHeaders,
auth=auth)
glogger.debug('Response header from endpoint: ' +
response.headers['Content-Type'])
# Response headers
resp = response.text
headers['Content-Type'] = response.headers['Content-Type']
# If there's no mime type, the endpoint is an actual SPARQL endpoint
else:
reqHeaders = {'Accept': acceptHeader}
if content:
reqHeaders = {'Accept': static.mimetypes[content]}
data = {'query': rewritten_query}
glogger.debug(
'Sending HTTP request to SPARQL endpoint with params: {}'.format(
data))
glogger.debug(
'Sending HTTP request to SPARQL endpoint with headers: {}'.format(
reqHeaders))
glogger.debug(
'Sending HTTP request to SPARQL endpoint with auth: {}'.format(
auth))
try:
response = requests.get(endpoint,
params=data,
headers=reqHeaders,
auth=auth)
except Exception as e:
# Error contacting SPARQL endpoint
glogger.debug(
'Exception encountered while connecting to SPARQL endpoint')
return {'error': str(e)}, 400, headers
glogger.debug('Response header from endpoint: ' +
response.headers['Content-Type'])
# Response headers
resp = response.text
headers['Content-Type'] = response.headers['Content-Type']
# If the query is paginated, set link HTTP headers
if pagination:
# Get number of total results
count = gquery.count_query_results(rewritten_query, endpoint)
pageArg = requestArgs.get('page', None)
url = urlparse(requestUrl)
replaced = url._replace(netloc=static.SERVER_NAME)
headerLink = pageUtils.buildPaginationHeader(count,
pagination, pageArg,
replaced.geturl())
headers['Link'] = headerLink
if 'proto' in query_metadata: # sparql transformer
resp = SPARQLTransformer.post_process(json.loads(resp),
query_metadata['proto'],
query_metadata['opt'])
if 'transform' in query_metadata and acceptHeader == 'application/json': # sparql transformer
rq = {'proto': query_metadata['transform']}
_, _, opt = SPARQLTransformer.pre_process(rq)
resp = SPARQLTransformer.post_process(json.loads(resp),
query_metadata['transform'], opt)
headers['Server'] = 'grlc/' + grlc_version
return resp, 200, headers
def process_sparql_query_text(query_text, loader, call_name, extraMetadata):
"""Generates a swagger specification item based on the given SPARQL query file."""
# We get the endpoint name first, since some query metadata fields (eg enums) require it
endpoint, _ = gquery.guess_endpoint_uri(query_text, loader)
glogger.debug("Read query endpoint: {}".format(endpoint))
try:
query_metadata = gquery.get_metadata(query_text, endpoint)
except Exception as e:
raise Exception('Could not parse query {}: {}'.format(call_name, str(e)))
tags = query_metadata['tags'] if 'tags' in query_metadata else []
summary = query_metadata['summary'] if 'summary' in query_metadata else ""
description = query_metadata['description'] if 'description' in query_metadata else ""
method = query_metadata['method'].lower() if 'method' in query_metadata else ""
if method not in ['get', 'post', 'head', 'put', 'delete', 'options', 'connect']:
method = ""
pagination = query_metadata['pagination'] if 'pagination' in query_metadata else ""
endpoint_in_url = query_metadata['endpoint_in_url'] if 'endpoint_in_url' in query_metadata else True
# Processing of the parameters
params = []
# PV properties
item_properties = {}
# If this query allows pagination, add page number as parameter
if pagination:
params.append(pageUtils.getSwaggerPaginationDef(pagination))
if query_metadata['type'] in ['SelectQuery', 'ConstructQuery', 'InsertData']:
# TODO: do something intelligent with the parameters!
# As per #3, prefetching IRIs via SPARQL and filling enum
parameters = query_metadata['parameters']
for _, p in list(parameters.items()):
param = {}
param['name'] = p['name']
param['type'] = p['type']
param['required'] = p['required']
param['in'] = "query"
param['description'] = "A value of type {} that will substitute {} in the original query".format(
p['type'], p['original'])
if 'lang' in p:
param['description'] = "A value of type {}@{} that will substitute {} in the original query".format(
p['type'], p['lang'], p['original'])
if 'format' in p:
param['format'] = p['format']
param['description'] = "A value of type {} ({}) that will substitute {} in the original query".format(
p['type'], p['format'], p['original'])
if 'enum' in p:
param['enum'] = p['enum']
if 'default' in p:
param['default'] = p['default']
params.append(param)
if endpoint_in_url:
endpoint_param = {}
endpoint_param['name'] = "endpoint"
endpoint_param['type'] = "string"
endpoint_param['in'] = "query"
endpoint_param['description'] = "Alternative endpoint for SPARQL query"
endpoint_param['default'] = endpoint
params.append(endpoint_param)
# If this is a URL generated spec we need to force API calls with the specUrl parameter set
if type(loader) is URLLoader:
specUrl_param = {}
specUrl_param['name'] = "specUrl"
specUrl_param['type'] = "string"
specUrl_param['in'] = "query"
specUrl_param['description'] = "URL of the API specification"
specUrl_param['default'] = loader.getRawRepoUri()
params.append(specUrl_param)
if query_metadata['type'] == 'SelectQuery':
# Fill in the spec for SELECT
if not method:
method = 'get'
for pv in query_metadata['variables']:
item_properties[pv] = {
"name": pv,
"type": "object",
"required": ["type", "value"],
"properties": {
"type": {
"type": "string"
},
"value": {
"type": "string"
},
"xml:lang": {
"type": "string"
},
"datatype": {
"type": "string"
}
}
}
elif query_metadata['type'] == 'ConstructQuery':
if not method:
method = 'get'
elif query_metadata['type'] == 'UNKNOWN':
glogger.warning("grlc could not parse this query; assuming a plain, non-parametric SELECT in the API spec")
if not method:
method = 'get'
else:
# TODO: process all other kinds of queries
raise Exception('Could not parse query {}: Query of type {} is currently unsupported'.format(call_name, query_metadata['type']))
# Finally: main structure of the callname spec
item = packItem('/' + call_name, method, tags, summary, description, params, query_metadata, extraMetadata)
return item
def process_sparql_query_text(query_text, loader, call_name, extraMetadata):
# We get the endpoint name first, since some query metadata fields (eg enums) require it
endpoint, auth = gquery.guess_endpoint_uri(query_text, loader)
glogger.debug("Read query endpoint: {}".format(endpoint))
try:
query_metadata = gquery.get_metadata(query_text, endpoint)
except Exception:
raw_repo_uri = loader.getRawRepoUri()
raw_query_uri = raw_repo_uri + ' / ' + call_name
glogger.error("Could not parse query at {}".format(raw_query_uri))
glogger.error(traceback.print_exc())
return None
tags = query_metadata['tags'] if 'tags' in query_metadata else []
summary = query_metadata['summary'] if 'summary' in query_metadata else ""
description = query_metadata['description'] if 'description' in query_metadata else ""
method = query_metadata['method'].lower() if 'method' in query_metadata else ""
if method not in ['get', 'post', 'head', 'put', 'delete', 'options', 'connect']:
method = ""
pagination = query_metadata['pagination'] if 'pagination' in query_metadata else ""
endpoint_in_url = query_metadata['endpoint_in_url'] if 'endpoint_in_url' in query_metadata else True
projection = loader.getProjectionForQueryName(call_name)
# Processing of the parameters
params = []
# PV properties
item_properties = {}
# If this query allows pagination, add page number as parameter
if pagination:
params.append(pageUtils.getSwaggerPaginationDef(pagination))
if query_metadata['type'] == 'SelectQuery' or query_metadata['type'] == 'ConstructQuery' or query_metadata['type'] == 'InsertData':
# TODO: do something intelligent with the parameters!
# As per #3, prefetching IRIs via SPARQL and filling enum
parameters = query_metadata['parameters']
for v, p in list(parameters.items()):
param = {}
param['name'] = p['name']
param['type'] = p['type']
param['required'] = p['required']
param['in'] = "query"
param['description'] = "A value of type {} that will substitute {} in the original query".format(p['type'], p['original'])
if 'lang' in p:
param['description'] = "A value of type {}@{} that will substitute {} in the original query".format(p['type'], p['lang'], p['original'])
if 'format' in p:
param['format'] = p['format']
param['description'] = "A value of type {} ({}) that will substitute {} in the original query".format(p['type'], p['format'], p['original'])
if 'enum' in p:
param['enum'] = p['enum']
if 'default' in p:
param['default'] = p['default']
params.append(param)
if endpoint_in_url:
endpoint_param = {}
endpoint_param['name'] = "endpoint"
endpoint_param['type'] = "string"
endpoint_param['in'] = "query"
endpoint_param['description'] = "Alternative endpoint for SPARQL query"
endpoint_param['default'] = endpoint
params.append(endpoint_param)
if query_metadata['type'] == 'SelectQuery':
# Fill in the spec for SELECT
if not method:
method = 'get'
for pv in query_metadata['variables']:
item_properties[pv] = {
"name": pv,
"type": "object",
"required": ["type", "value"],
"properties": {
"type": {
"type": "string"
},
"value": {
"type": "string"
},
"xml:lang": {
"type": "string"
},
"datatype": {
"type": "string"
}
}
}
elif query_metadata['type'] == 'ConstructQuery':
if not method:
method = 'get'
elif query_metadata['type'] == 'UNKNOWN':
glogger.warning("grlc could not parse this query; assuming a plain, non-parametric SELECT in the API spec")
if not method:
method = 'get'
else:
# TODO: process all other kinds of queries
glogger.warning("Query of type {} is currently unsupported! Skipping".format(query_metadata['type']))
# Finally: main structure of the callname spec
item = packItem('/' + call_name, method, tags, summary, description, params, query_metadata, extraMetadata, projection)
return item