def _handle_query_result(self, result: QueryResult) -> Optional[SqlError]:
query = self.get_query_by_task_id(result.query_task_id)
if result.status in ("complete", "error"):
self._running_queries.remove(query)
self.query_slots += 1
lookml_object = query.lookml_ref
lookml_object.queried = True
if result.status == "error" and result.error:
model_name = lookml_object.model_name
dimension_name: Optional[str] = None
if isinstance(lookml_object, Dimension):
explore_name = lookml_object.explore_name
dimension_name = lookml_object.name
else:
explore_name = lookml_object.name
sql_error = SqlError(
model=model_name,
explore=explore_name,
dimension=dimension_name,
explore_url=query.explore_url,
lookml_url=getattr(lookml_object, "url", None),
**result.error,
)
lookml_object.error = sql_error
return sql_error
return None
def sql_error():
return SqlError(
dimension="users.age",
explore="users",
model="eye_exam",
sql="SELECT age FROM users WHERE 1=2 LIMIT 1",
message="An error occurred.",
explore_url="https://spectacles.looker.com/x/qCJsodAZ2Y22QZLbmD0Gvy",
)
def _get_query_results(
self,
query_task_ids: List[str]) -> Tuple[List[str], List[SqlError]]:
results = self.client.get_query_task_multi_results(query_task_ids)
still_running = []
errors = []
for query_task_id, query_result in results.items():
query_status = query_result["status"]
logger.debug("Query task %s status is %s", query_task_id,
query_status)
if query_status in ("running", "added", "expired"):
still_running.append(query_task_id)
continue
elif query_status in ("complete", "error"):
lookml_object = self.query_tasks[query_task_id]
lookml_object.queried = True
else:
raise SpectaclesException(
f'Unexpected query result status "{query_status}" '
"returned by the Looker API")
if query_status == "error":
try:
details = self._extract_error_details(query_result)
except (KeyError, TypeError, IndexError) as error:
raise SpectaclesException(
"Encountered an unexpected API query result format, "
"unable to extract error details. "
f"The query result was: {query_result}") from error
sql_error = SqlError(
path=lookml_object.name,
url=getattr(lookml_object, "url", None),
**details,
)
lookml_object.error = sql_error
errors.append(sql_error)
return still_running, errors
async def _get_query_results(
self, session: aiohttp.ClientSession) -> List[SqlError]:
logger.debug("%d queries running", self.running_query_tasks.qsize())
try:
# Empty the queue (up to 250) to get all running query tasks
query_task_ids: List[str] = []
while not self.running_query_tasks.empty() and len(
query_task_ids) <= 250:
query_task_ids.append(await self.running_query_tasks.get())
logger.debug("Getting results for %d query tasks",
len(query_task_ids))
results = await self.client.get_query_task_multi_results(
session, query_task_ids)
pending_task_ids = []
errors = []
for query_task_id, query_result in results.items():
query_status = query_result["status"]
logger.debug("Query task %s status is %s", query_task_id,
query_status)
if query_status in ("running", "added", "expired"):
pending_task_ids.append(query_task_id)
# Put the running query tasks back in the queue
await self.running_query_tasks.put(query_task_id)
query_task_ids.remove(query_task_id)
continue
elif query_status in ("complete", "error"):
query_task_ids.remove(query_task_id)
# We can release a query slot for each completed query
self.query_slots.release()
lookml_object = self.query_tasks[query_task_id]
lookml_object.queried = True
if query_status == "error":
try:
details = self._extract_error_details(query_result)
except (KeyError, TypeError, IndexError) as error:
raise SpectaclesException(
"Encountered an unexpected API query result format, "
"unable to extract error details. "
f"The query result was: {query_result}"
) from error
sql_error = SqlError(
path=lookml_object.name,
url=getattr(lookml_object, "url", None),
**details,
)
lookml_object.error = sql_error
errors.append(sql_error)
else:
raise SpectaclesException(
f'Unexpected query result status "{query_status}" '
"returned by the Looker API")
except asyncio.CancelledError:
logger.debug(
"Cancelled result fetching, putting "
f"{self.running_query_tasks.qsize()} query task IDs back in the queue"
)
for query_task_id in query_task_ids:
await self.running_query_tasks.put(query_task_id)
logger.debug("Restored query task IDs to queue")
raise
return errors