def query_results(owner, app_name, job_id):
cluster_type = get_cluster_type(owner, app_name)
if cluster_type == "single-core":
async_result = AsyncResult(job_id)
print("celery result", async_result.state)
if async_result.ready() and async_result.successful():
return "YES"
elif async_result.failed():
return "FAIL"
else:
return "NO"
elif cluster_type == "dask":
addr = dask_scheduler_address(owner, app_name)
with Client(addr) as client:
fut = Future(job_id, client=client)
print("dask result", fut.status)
if fut.done() and fut.status != "error":
return "YES"
elif fut.done() and fut.status in ("error", "cancelled"):
return "FAIL"
else:
return "NO"
else:
return json.dumps({"error": "model does not exist."}), 404
def results(owner, app_name, job_id):
cluster_type = get_cluster_type(owner, app_name)
if cluster_type == "single-core":
async_result = AsyncResult(job_id)
if async_result.ready() and async_result.successful():
return json.dumps(async_result.result)
elif async_result.failed():
print("traceback", async_result.traceback)
return json.dumps({
"status": "WORKER_FAILURE",
"traceback": async_result.traceback
})
else:
return make_response("not ready", 202)
elif cluster_type == "dask":
addr = dask_scheduler_address(owner, app_name)
with Client(addr) as client:
fut = Future(job_id, client=client)
if fut.done() and fut.status != "error":
return fut.result()
elif fut.done() and fut.status in ("error", "cancelled"):
return json.dumps({
"status": "WORKER_FAILURE",
"traceback": fut.traceback()
})
else:
return make_response("not ready", 202)
else:
return json.dumps({"error": "model does not exist."}), 404
def _process_future(self, future: Future) -> None:
if not self.futures:
raise AirflowException(NOT_STARTED_MESSAGE)
if future.done():
key = self.futures[future]
if future.exception():
self.log.error("Failed to execute task: %s", repr(future.exception()))
self.fail(key)
elif future.cancelled():
self.log.error("Failed to execute task")
self.fail(key)
else:
self.success(key)
self.futures.pop(future)