def execute_plan(self, interpreter):
""" Execute a query using a schema based query planning strategy. """
self.service = ''
plan = self.planner.plan(self.query)
statements = self.plan(plan)
responses = []
for index, statement in enumerate(statements):
logger.debug(f" -- {statement.query}")
response = statement.execute(interpreter)
responses.append(response)
if index < len(statements) - 1:
""" Implement handoff. Finds the type name of the first element of the
next plan segment, looks up values for that type from the answer bindings of the
last response, and transfers values to the new question. TODO: incorporate
user specified namnes. """
next_statement = statements[index + 1]
name = next_statement.query.order[0]
#name = statement.query.order[-1]
#values = self.jsonkit.select (f"$.knowledge_map.[*].node_bindings.{name}", response)
values = self.jsonkit.select(
f"$.knowledge_map.[*].[*].node_bindings.{name}", response)
first_concept = next_statement.query.concepts[name]
first_concept.set_nodes(values)
if len(values) == 0:
raise ServiceInvocationError(
f"No valid results from service {statement.service} executing "
+
f"query {statement.query}. Unable to continue query. Exiting."
)
merged = self.merge_results(responses, self.service)
questions = self.generate_questions(interpreter)
merged['question_graph'] = questions[0]['question_graph']
return merged
def make_request_sync(**kwargs):
response = {}
unknown_service = False
try:
http_response = requests.request(**kwargs)
""" Check status and handle response. """
if http_response.status_code == 200 or http_response.status_code == 202:
response = http_response.json()
#logger.error (f" response: {json.dumps(response, indent=2)}")
status = response.get('status', None)
if status == "error":
raise ServiceInvocationError(
message=f"An error occurred invoking service: {url}.",
details=truncate(response['message'], max_length=5000))
elif http_response.status_code == 404:
unknown_service = True
else:
pass
# logger.error (f"error {http_response.status_code} processing request: {message}")
# logger.error (http_response.text)
except ServiceInvocationError as e:
raise e
except Exception as e:
print(e)
# logger.error (f"error performing request: {json.dumps(message, indent=2)} to url: {url}")
#traceback.print_exc ()
# logger.error (traceback.format_exc ())
if unknown_service:
raise UnknownServiceError(
f"Service {url} was not found. Is it misspelled?")
return response
async def make_request_async(semaphore, **kwargs):
response = {}
errors = []
async with aiohttp.ClientSession() as session:
try:
async with session.request(**kwargs) as http_response:
# print(f"[{kwargs['method'].upper()}] requesting at url: {kwargs['url']}")
""" Check status and handle response. """
if http_response.status == 200 or http_response.status == 202:
response = await http_response.json()
#logger.error (f" response: {json.dumps(response, indent=2)}")
status = response.get('status', None)
if status == "error":
raise ServiceInvocationError(
f"An error occurred invoking service: {kwargs['url']}.",
response['message'])
elif http_response.status == 404:
raise UnknownServiceError(
f"Service {url} was not found. Is it misspelled?")
else:
http_response.raise_for_status()
# logger.error (f"error {http_response.status} processing request: {message}")
# logger.error (http_response.text)
except concurrent.futures.TimeoutError as e:
errors.append(
RequestTimeoutError(
f'Timeout error requesting content from url: "{kwargs.get("url","undefined")}"',
kwargs))
except ServiceInvocationError as e:
errors.append(e)
except Exception as e:
errors.append(e)
return {"response": response, "errors": errors}
def get_ids(self, name, type_name):
url = self.url.format(**{"input": name, "type": type_name})
result = None
response = requests.get(url=url,
headers={'accept': 'application/json'})
if response.status_code == 200 or response.status_code == 202:
result = response.json()
else:
raise ServiceInvocationError(response.text)
return result
def execute(self, interpreter, context={}):
"""
Execute all statements in the abstract syntax tree.
- Generate questions by permuting bound values.
- Resolve the service name.
- Execute the questions.
"""
result = None
if self.service == "/schema":
result = self.execute_plan(interpreter)
else:
self.service = self.resolve_backplane_url(self.service,
interpreter)
questions = self.generate_questions(interpreter)
service = interpreter.context.resolve_arg(self.service)
""" Invoke the service and store the response. """
responses = []
for index, q in enumerate(questions):
logger.debug(f"executing question {json.dumps(q, indent=2)}")
response = self.request(service, q)
# TODO - add a parameter to limit service invocations.
# Until we parallelize requests, cap the max number we attempt for performance reasons.
if index > 50:
break
#logger.debug (f"response: {json.dumps(response, indent=2)}")
responses.append(response)
if len(responses) == 0:
raise ServiceInvocationError(
f"No valid results from service {self.service} executing "
+
f"query {self.query}. Unable to continue query. Exiting.")
#raise ServiceInvocationError (f"No responses received from {service}")
result = self.merge_results(responses, service)
interpreter.context.set('result', result)
""" Execute set statements associated with this statement. """
for set_statement in self.set_statements:
logger.debug(f"{set_statement}")
set_statement.execute(interpreter, context={"result": result})
return result
def request(self, url, message):
""" Make a web request to a service (url) posting a message. """
logger.debug(f"request({url})> {json.dumps(message, indent=2)}")
response = {}
unknown_service = False
try:
http_response = requests.post(
url=url, json=message, headers={'accept': 'application/json'})
""" Check status and handle response. """
if http_response.status_code == 200 or http_response.status_code == 202:
response = http_response.json()
#logger.error (f" response: {json.dumps(response, indent=2)}")
status = response.get('status', None)
if status == "error":
raise ServiceInvocationError(
message=f"An error occurred invoking service: {url}.",
details=truncate(response['message'], max_length=5000))
logging.debug(f"{json.dumps(response, indent=2)}")
elif http_response.status_code == 404:
unknown_service = True
else:
logger.error(
f"error {http_response.status_code} processing request: {message}"
)
logger.error(http_response.text)
except ServiceInvocationError as e:
raise e
except Exception as e:
logger.error(
f"error performing request: {json.dumps(message, indent=2)} to url: {url}"
)
#traceback.print_exc ()
logger.error(traceback.format_exc())
if unknown_service:
raise UnknownServiceError(
f"Service {url} was not found. Is it misspelled?")
return response
def make_onto_request(url):
response = requests.get(url, headers={'accept': 'application/json'})
if response.ok:
return response.json()
else:
raise ServiceInvocationError(response.text)
def execute(self, interpreter, context={}):
"""
Execute all statements in the abstract syntax tree.
- Generate questions by permuting bound values.
- Resolve the service name.
- Execute the questions.
"""
result = None
if self.service == "/schema":
result = self.execute_plan(interpreter)
else:
self.service = self.resolve_backplane_url(self.service,
interpreter)
questions = self.generate_questions(interpreter)
[
self.ast.schema.validate_question(question)
for question in questions
]
service = interpreter.context.resolve_arg(self.service)
""" Invoke the service and store the response. """
# For each question, make a request to the service with the question
# Only have a maximum of maximumParallelRequests requests executing at any given time
logger.setLevel(logging.DEBUG)
logger.debug(
f"Starting queries on service: {service} (asynchronous={interpreter.asynchronous})"
)
logger.setLevel(logging.INFO)
prev = time.time()
# We don't want to flood the service so we cap the maximum number of requests we can make to it.
maximumQueryRequests = 50
interpreter.context.set('requestErrors', [])
if interpreter.asynchronous:
maximumParallelRequests = 4
responses = async_make_requests([{
"method": "post",
"url": service,
"json": q,
"headers": {
"accept": "application/json"
}
} for q in questions[:maximumQueryRequests]],
maximumParallelRequests)
errors = responses["errors"]
responses = responses["responses"]
interpreter.context.mem.get('requestErrors', []).extend(errors)
else:
responses = []
for index, q in enumerate(questions):
logger.debug(
f"executing question {json.dumps(q, indent=2)}")
response = self.request(service, q)
# TODO - add a parameter to limit service invocations.
# Until we parallelize requests, cap the max number we attempt for performance reasons.
if index >= maximumQueryRequests:
break
#logger.debug (f"response: {json.dumps(response, indent=2)}")
responses.append(response)
logger.setLevel(logging.DEBUG)
logger.debug(
f"Making requests took {time.time()-prev} s (asynchronous = {interpreter.asynchronous})"
)
logger.setLevel(logging.INFO)
if len(responses) == 0:
# interpreter.context.mem.get('requestErrors',[]).append(ServiceInvocationError(
# f"No valid results from {self.service} with query {self.query}"
# ))
raise ServiceInvocationError(
f"No valid results from service {self.service} executing "
+
f"query {self.query}. Unable to continue query. Exiting.")
result = self.merge_results(responses, service, interpreter)
interpreter.context.set('result', result)
""" Execute set statements associated with this statement. """
for set_statement in self.set_statements:
logger.debug(f"{set_statement}")
set_statement.execute(interpreter, context={"result": result})
return result