def process_tpf_query_text(query_text, raw_repo_uri, call_name, extraMetadata):
query_metadata = gquery.get_yaml_decorators(query_text)
tags = query_metadata['tags'] if 'tags' in query_metadata else []
glogger.debug("Read query tags: " + ', '.join(tags))
summary = query_metadata['summary'] if 'summary' in query_metadata else ""
glogger.debug("Read query summary: " + summary)
description = query_metadata[
'description'] if 'description' in query_metadata else ""
glogger.debug("Read query description: " + description)
method = query_metadata['method'].lower(
) if 'method' in query_metadata else "get"
if method not in [
'get', 'post', 'head', 'put', 'delete', 'options', 'connect'
]:
method = "get"
pagination = query_metadata[
'pagination'] if 'pagination' in query_metadata else ""
glogger.debug("Read query pagination: " + str(pagination))
# enums = query_metadata['enumerate'] if 'enumerate' in query_metadata else []
# glogger.debug("Read query enumerates: " + ', '.join(enums))
endpoint = query_metadata[
'endpoint'] if 'endpoint' in query_metadata else ""
glogger.debug("Read query endpoint: " + endpoint)
# If this query allows pagination, add page number as parameter
params = []
if pagination:
pagination_param = {}
pagination_param['name'] = "page"
pagination_param['type'] = "int"
pagination_param['in'] = "query"
pagination_param[
'description'] = "The page number for this paginated query ({} results per page)".format(
pagination)
params.append(pagination_param)
item = {
'call_name': call_name,
'method': method,
'tags': tags,
'summary': summary,
'description': description,
'params': params,
'query': query_metadata['query']
}
for extraField in extraMetadata:
if extraField in query_metadata:
item[extraField] = query_metadata[extraField]
return item
def process_tpf_query_text(resp, raw_repo_uri, call_name, extraMetadata):
query_metadata = gquery.get_yaml_decorators(resp)
tags = query_metadata['tags'] if 'tags' in query_metadata else []
glogger.debug("Read query tags: " + ', '.join(tags))
summary = query_metadata['summary'] if 'summary' in query_metadata else ""
glogger.debug("Read query summary: " + summary)
description = query_metadata['description'] if 'description' in query_metadata else ""
glogger.debug("Read query description: " + description)
method = query_metadata['method'].lower() if 'method' in query_metadata else "get"
if method not in ['get', 'post', 'head', 'put', 'delete', 'options', 'connect']:
method = "get"
pagination = query_metadata['pagination'] if 'pagination' in query_metadata else ""
glogger.debug("Read query pagination: " + str(pagination))
# enums = query_metadata['enumerate'] if 'enumerate' in query_metadata else []
# glogger.debug("Read query enumerates: " + ', '.join(enums))
endpoint = query_metadata['endpoint'] if 'endpoint' in query_metadata else ""
glogger.debug("Read query endpoint: " + endpoint)
# If this query allows pagination, add page number as parameter
if pagination:
pagination_param = {}
pagination_param['name'] = "page"
pagination_param['type'] = "int"
pagination_param['in'] = "query"
pagination_param['description'] = "The page number for this paginated query ({} results per page)".format(pagination)
params.append(pagination_param)
item = {
'call_name': call_name,
'method': method,
'tags': tags,
'summary': summary,
'description': description,
'query': query_metadata['query']
}
for extraField in extraMetadata:
if extraField in query_metadata:
item[extraField] = query_metadata[extraField]
return item
def query(user, repo, query_name, content=None):
glogger.debug("Got request at endpoint /" + user + "/" + repo + "/" + query_name)
glogger.debug("Request accept header: " + request.headers["Accept"])
raw_repo_uri = static.GITHUB_RAW_BASE_URL + user + "/" + repo + "/master/"
# The URIs of all candidates
raw_sparql_query_uri = raw_repo_uri + query_name + ".rq"
raw_alt_sparql_query_uri = raw_repo_uri + query_name + ".sparql"
raw_tpf_query_uri = raw_repo_uri + query_name + ".tpf"
raw_sparql_query = requests.get(raw_sparql_query_uri)
raw_alt_sparql_query = requests.get(raw_alt_sparql_query_uri)
raw_tpf_query = requests.get(raw_tpf_query_uri)
# Call name implemented with SPARQL query
if raw_sparql_query.status_code == 200 or raw_alt_sparql_query.status_code == 200:
if raw_sparql_query.status_code == 200:
raw_sparql_query = raw_sparql_query.text
else:
raw_sparql_query = raw_alt_sparql_query.text
endpoint = gquery.guess_endpoint_uri(raw_sparql_query, raw_repo_uri)
glogger.debug("=====================================================")
glogger.debug("Sending query to SPARQL endpoint: {}".format(endpoint))
glogger.debug("=====================================================")
query_metadata = gquery.get_metadata(raw_sparql_query)
pagination = query_metadata["pagination"] if "pagination" in query_metadata else ""
# Rewrite query using parameter values
rewritten_query = gquery.rewrite_query(raw_sparql_query, request.args, endpoint)
# Rewrite query using pagination
paginated_query = gquery.paginate_query(rewritten_query, request.args)
resp = None
# If we have a mime field, we load the remote dump and query it locally
if "mime" in query_metadata and query_metadata["mime"]:
g = Graph()
try:
query_metadata = gquery.get_metadata(raw_sparql_query)
g.parse(endpoint, format=query_metadata["mime"])
except Exception as e:
glogger.error(e)
results = g.query(paginated_query, result="sparql")
# glogger.debug("Results of SPARQL query against locally loaded dump:")
# Prepare return format as requested
resp_string = ""
# glogger.debug("Requested formats: {}".format(request.headers['Accept']))
# if content:
# glogger.debug("Requested formats from extension: {}".format(static.mimetypes[content]))
if "application/json" in request.headers["Accept"] or (
content and "application/json" in static.mimetypes[content]
):
resp_string = results.serialize(format="json")
elif "text/csv" in request.headers["Accept"] or (content and "text/csv" in static.mimetypes[content]):
resp_string = results.serialize(format="csv")
# elif 'text/html' in request.headers['Accept']:
# resp_string = results.serialize(format='html')
else:
return "Unacceptable requested format", 415
del g
resp = make_response(resp_string)
# If there's no mime type, the endpoint is an actual SPARQL endpoint
else:
# Preapre HTTP request
headers = {"Accept": request.headers["Accept"]}
if content:
headers = {"Accept": static.mimetypes[content]}
data = {"query": paginated_query}
response = requests.get(endpoint, params=data, headers=headers)
glogger.debug("Response header from endpoint: " + response.headers["Content-Type"])
# Response headers
resp = make_response(response.text)
resp.headers["Server"] = "grlc/1.0.0"
resp.headers["Content-Type"] = request.headers["Content-Type"]
# If the query is paginated, set link HTTP headers
if pagination:
# Get number of total results
count = gquery.count_query_results(rewritten_query, endpoint)
page = 1
if "page" in request.args:
page = int(request.args["page"])
next_url = re.sub("page=[0-9]+", "page={}".format(page + 1), request.url)
prev_url = re.sub("page=[0-9]+", "page={}".format(page - 1), request.url)
first_url = re.sub("page=[0-9]+", "page=1", request.url)
last_url = re.sub("page=[0-9]+", "page={}".format(count / pagination), request.url)
else:
next_url = request.url + "?page={}".format(page + 1)
prev_url = request.url + "?page={}".format(page - 1)
first_url = request.url + "?page={}".format(page)
last_url = request.url + "?page={}".format(count / pagination)
if page == 1:
resp.headers["Link"] = "<{}>; rel=next, <{}>; rel=last".format(next_url, last_url)
elif page == count / pagination:
resp.headers["Link"] = "<{}>; rel=prev, <{}>; rel=first".format(prev_url, first_url)
else:
resp.headers["Link"] = "<{}>; rel=next, <{}>; rel=prev, <{}>; rel=first, <{}>; rel=last".format(
next_url, prev_url, first_url, last_url
)
return resp
# Call name implemented with TPF query
elif raw_tpf_query.status_code == 200:
raw_tpf_query = raw_tpf_query.text
endpoint = gquery.guess_endpoint_uri(raw_tpf_query, raw_repo_uri)
glogger.debug("=====================================================")
glogger.debug("Sending query to TPF endpoint: {}".format(endpoint))
glogger.debug("=====================================================")
query_metadata = gquery.get_yaml_decorators(raw_tpf_query)
# TODO: pagination for TPF
# Preapre HTTP request
headers = {"Accept": request.headers["Accept"]}
if content:
headers = {"Accept": static.mimetypes[content]}
tpf_list = re.split("\n|=", raw_tpf_query)
subject = tpf_list[tpf_list.index("subject") + 1]
predicate = tpf_list[tpf_list.index("predicate") + 1]
object = tpf_list[tpf_list.index("object") + 1]
data = {"subject": subject, "predicate": predicate, "object": object}
response = requests.get(endpoint, params=data, headers=headers)
glogger.debug("Response header from endpoint: " + response.headers["Content-Type"])
# Response headers
resp = make_response(response.text)
resp.headers["Server"] = "grlc/1.0.0"
resp.headers["Content-Type"] = response.headers["Content-Type"]
return resp
else:
return "Couldn't find a SPARQL, RDF dump, or TPF query with the requested name", 404
def query(user, repo, query_name, sha=None, content=None):
glogger.debug(
"-----> Executing call name at /{}/{}/{} on commit {}".format(
user, repo, query_name, sha))
glogger.debug("Request accept header: " + request.headers["Accept"])
if user is None and repo is None:
loader = LocalLoader()
else:
loader = GithubLoader(user, repo, sha, None)
query, q_type = loader.getTextForName(query_name)
# Call name implemented with SPARQL query
if q_type == qType['SPARQL']:
raw_sparql_query = query
raw_repo_uri = loader.getRawRepoUri()
endpoint, auth = gquery.guess_endpoint_uri(raw_sparql_query, loader)
glogger.debug("=====================================================")
glogger.debug("Sending query to SPARQL endpoint: {}".format(endpoint))
glogger.debug("=====================================================")
query_metadata = gquery.get_metadata(raw_sparql_query)
pagination = query_metadata[
'pagination'] if 'pagination' in query_metadata else ""
# Rewrite query using parameter values
rewritten_query = gquery.rewrite_query(raw_sparql_query, request.args,
endpoint)
# Rewrite query using pagination
paginated_query = gquery.paginate_query(rewritten_query, request.args)
resp = None
# If we have a mime field, we load the remote dump and query it locally
if 'mime' in query_metadata and query_metadata['mime']:
g = Graph()
try:
query_metadata = gquery.get_metadata(raw_sparql_query)
g.parse(endpoint, format=query_metadata['mime'])
except Exception as e:
glogger.error(e)
results = g.query(paginated_query, result='sparql')
# glogger.debug("Results of SPARQL query against locally loaded dump:")
# Prepare return format as requested
resp_string = ""
# glogger.debug("Requested formats: {}".format(request.headers['Accept']))
# if content:
# glogger.debug("Requested formats from extension: {}".format(static.mimetypes[content]))
if 'application/json' in request.headers['Accept'] or (
content
and 'application/json' in static.mimetypes[content]):
resp_string = results.serialize(format='json')
elif 'text/csv' in request.headers['Accept'] or (
content and 'text/csv' in static.mimetypes[content]):
resp_string = results.serialize(format='csv')
# elif 'text/html' in request.headers['Accept']:
# resp_string = results.serialize(format='html')
else:
return 'Unacceptable requested format', 415
del g
resp = make_response(resp_string)
# If there's no mime type, the endpoint is an actual SPARQL endpoint
else:
# Prepare HTTP request
headers = {'Accept': request.headers['Accept']}
if content:
headers = {
'Accept': static.mimetypes[content],
'Authorization': 'token {}'.format(static.ACCESS_TOKEN)
}
data = {'query': paginated_query}
response = requests.get(endpoint,
params=data,
headers=headers,
auth=auth)
glogger.debug('Response header from endpoint: ' +
response.headers['Content-Type'])
# Response headers
resp = make_response(response.text)
resp.headers['Server'] = 'grlc/1.0.0'
resp.headers['Content-Type'] = response.headers['Content-Type']
# If the query is paginated, set link HTTP headers
if pagination:
# Get number of total results
count = gquery.count_query_results(rewritten_query, endpoint)
page = 1
if 'page' in request.args:
page = int(request.args['page'])
next_url = re.sub("page=[0-9]+", "page={}".format(page + 1),
request.url)
prev_url = re.sub("page=[0-9]+", "page={}".format(page - 1),
request.url)
first_url = re.sub("page=[0-9]+", "page=1", request.url)
last_url = re.sub("page=[0-9]+",
"page={}".format(count / pagination),
request.url)
else:
next_url = request.url + "?page={}".format(page + 1)
prev_url = request.url + "?page={}".format(page - 1)
first_url = request.url + "?page={}".format(page)
last_url = request.url + "?page={}".format(count / pagination)
if page == 1:
resp.headers['Link'] = "<{}>; rel=next, <{}>; rel=last".format(
next_url, last_url)
elif page == count / pagination:
resp.headers[
'Link'] = "<{}>; rel=prev, <{}>; rel=first".format(
prev_url, first_url)
else:
resp.headers[
'Link'] = "<{}>; rel=next, <{}>; rel=prev, <{}>; rel=first, <{}>; rel=last".format(
next_url, prev_url, first_url, last_url)
return resp
# Call name implemented with TPF query
elif q_type == qType['TPF']:
raw_tpf_query = query
endpoint, auth = gquery.guess_endpoint_uri(raw_tpf_query, raw_repo_uri)
glogger.debug("=====================================================")
glogger.debug("Sending query to TPF endpoint: {}".format(endpoint))
glogger.debug("=====================================================")
query_metadata = gquery.get_yaml_decorators(raw_tpf_query)
# TODO: pagination for TPF
# Preapre HTTP request
headers = {
'Accept': request.headers['Accept'],
'Authorization': 'token {}'.format(static.ACCESS_TOKEN)
}
if content:
headers = {
'Accept': static.mimetypes[content],
'Authorization': 'token {}'.format(static.ACCESS_TOKEN)
}
tpf_list = re.split('\n|=', raw_tpf_query)
subject = tpf_list[tpf_list.index('subject') + 1]
predicate = tpf_list[tpf_list.index('predicate') + 1]
object = tpf_list[tpf_list.index('object') + 1]
data = {'subject': subject, 'predicate': predicate, 'object': object}
response = requests.get(endpoint,
params=data,
headers=headers,
auth=auth)
glogger.debug('Response header from endpoint: ' +
response.headers['Content-Type'])
# Response headers
resp = make_response(response.text)
resp.headers['Server'] = 'grlc/1.0.0'
resp.headers['Content-Type'] = response.headers['Content-Type']
return resp
else:
return "Couldn't find a SPARQL, RDF dump, or TPF query with the requested name", 404
