def handle_summary(self, url):
request = urllib.request.Request(url,
method="GET",
headers={
"Content-Type":
"application/json",
})
response = urllib.request.urlopen(request)
returns = response.read()
summary_result = {"summary": str(returns.decode())}
return execution.ExecutionResult(events=[summary_result], )
def handle(self, buffer, finished):
base_url = self.deployment.runtime_status["api_url"]
if self.context.method.name == "fit":
url = urllib.parse.urljoin(base_url, "fit")
elif self.context.method.name == "apply":
url = urllib.parse.urljoin(base_url, "apply")
elif self.context.method.name == "summary":
url = urllib.parse.urljoin(base_url, "summary")
return self.handle_summary(url)
else:
raise Exception("unsupported method")
# include a header and only keep the first len(self.context.fields) values
field_count = len(self.context.fields)
clean_buffer = io.BytesIO()
clean_buffer.write((','.join(self.context.fields) + "\n").encode())
while buffer.readable():
line = buffer.readline()
if not line:
break
clean_buffer.write(line[:-(field_count + 1)] + b"\n")
buffer_bytes = clean_buffer.getvalue()
self.logger.warning("sending %s bytes" % len(buffer_bytes))
content = {
"data": buffer_bytes.decode(),
"meta": {
"options": {
"params": {
"algo": "notebook",
}
}
},
}
for k, v in self.context.params.items():
if k == "model_name":
content["meta"]["options"]["model_name"] = v
if k == "feature_variables" or k == "target_variables":
v = v.split(',')
content["meta"][k] = v
content["meta"]["options"]["params"][k] = v
request = urllib.request.Request(url,
method="POST",
data=json.dumps(content).encode(),
headers={
"Content-Type":
"application/json",
})
response = urllib.request.urlopen(request)
returns = response.read()
result = json.loads(returns.decode())
if not "status" in result:
return execution.ExecutionResult(
error="No status found in container results")
status = result['status']
if status == "error":
return execution.ExecutionResult(error=result['message'])
return_events = []
if "results" in result:
return_events = result['results']
return execution.ExecutionResult(events=return_events, )
def handle(self, buffer, finished):
if buffer:
self.send_events(buffer)
if self.context.is_preop:
return
if not finished:
return
if not self.spark_app_is_ready:
spark_app = self.get_spark_app()
if not "status" in spark_app:
self.logger.warning(
"could not find 'status' in spark app object")
else:
spark_status = spark_app["status"]
spark_application_running = False
if not "applicationState" in spark_status:
self.logger.warning(
"could not find 'applicationState' in spark app status"
)
else:
spark_application_state = spark_status["applicationState"]
if not "state" in spark_application_state:
self.logger.warning(
"could not find 'state' in 'applicationState'")
else:
spark_application_state_state = spark_application_state[
"state"]
self.logger.warning("spark application state: %s" %
spark_application_state_state)
if spark_application_state_state == "FAILED":
if "driverInfo" in spark_status:
driver_info = spark_status["driverInfo"]
if "podName" in driver_info:
driver_pod_name = driver_info["podName"]
self.logger.warning(
"spark driver pod name: %s" %
driver_pod_name)
try:
driver_logs = self.deployment.core_api.read_namespaced_pod_log(
name=driver_pod_name,
namespace=self.deployment.
environment.namespace,
tail_lines=100,
container="spark-kubernetes-driver",
)
self.logger.warning(
"spark driver logs: %s" %
driver_logs)
except:
self.logger.warning(
"could not read spark driver logs")
try:
driver_pod_status = self.deployment.get_pod_status(
driver_pod_name)
self.logger.warning(
"spark driver pod status: %s" %
driver_pod_status)
except:
self.logger.warning(
"could not read spark driver pod status"
)
if "errorMessage" in spark_application_state:
raise execution.UserFriendlyError(
"Spark failed: %s" %
spark_application_state["errorMessage"])
raise execution.UserFriendlyError(
"Spark failed: %s" % spark_application_state)
if spark_application_state_state == "RUNNING":
spark_application_running = True
spark_executors_running = False
if not "executorState" in spark_status:
self.logger.warning(
"could not find 'executorState' in spark app status")
else:
spark_executor_state = spark_status["executorState"]
number_of_running_executors = 0
for executor_name, executor_state in spark_executor_state.items(
):
self.logger.warning("executor %s: %s" %
(executor_name, executor_state))
if executor_state == "RUNNING":
number_of_running_executors += 1
if number_of_running_executors == self.executor_instance_count:
spark_executors_running = True
self.spark_app_is_ready = spark_application_running and spark_executors_running
if not self.spark_app_is_ready:
return execution.ExecutionResult(
wait=1,
final=False,
)
if not self.sent_start_signal:
self.signal_start()
self.sent_start_signal = True
if not self.output_completed:
status = self.get_status()
if status["status"] == "running":
pass
elif status["status"] == "error":
self.signal_stop()
return execution.ExecutionResult(error=status["error"])
elif status["status"] == "completed":
self.signal_stop()
self.output_completed = True
else:
return execution.ExecutionResult(
error="unexpected status: %s" % status["status"])
received_events = self.receive_events()
#self.logger.warning("received_events: %s" % received_events)
if received_events:
return execution.ExecutionResult(
events=received_events,
final=False,
)
if not self.output_completed:
return execution.ExecutionResult(
wait=1,
final=False,
)