def predict(deployment_name, api_name):
try:
payload = request.get_json()
except Exception as e:
return "Malformed JSON", status.HTTP_400_BAD_REQUEST
ctx = local_cache["ctx"]
api = local_cache["api"]
response = {}
if not util.is_dict(payload) or "samples" not in payload:
util.log_pretty_flat(payload, logging_func=logger.error)
return prediction_failed(
payload, "top level `samples` key not found in request")
predictions = []
samples = payload["samples"]
if not util.is_list(samples):
util.log_pretty_flat(samples, logging_func=logger.error)
return prediction_failed(
payload,
"expected the value of key `samples` to be a list of json objects")
for i, sample in enumerate(payload["samples"]):
try:
result = run_predict(sample)
except CortexException as e:
e.wrap("error", "sample {}".format(i + 1))
logger.error(str(e))
logger.exception(
"An error occurred, see `cortex logs -v api {}` for more details."
.format(api["name"]))
return prediction_failed(sample, str(e))
except Exception as e:
logger.exception(
"An error occurred, see `cortex logs -v api {}` for more details."
.format(api["name"]))
return prediction_failed(sample, str(e))
predictions.append(result)
response["predictions"] = predictions
response["resource_id"] = api["id"]
return jsonify(response)
def create_raw_prediction_request(sample):
signature_def = local_cache["metadata"]["signatureDef"]
signature_key = list(signature_def.keys())[0]
prediction_request = predict_pb2.PredictRequest()
prediction_request.model_spec.name = "default"
prediction_request.model_spec.signature_name = signature_key
for column_name, value in sample.items():
shape = [1]
if util.is_list(value):
shape = [len(value)]
sig_type = signature_def[signature_key]["inputs"][column_name]["dtype"]
tensor_proto = tf.make_tensor_proto([value],
dtype=DTYPE_TO_TF_TYPE[sig_type],
shape=shape)
prediction_request.inputs[column_name].CopyFrom(tensor_proto)
return prediction_request
def create_prediction_request(transformed_sample):
ctx = local_cache["ctx"]
signature_def = local_cache["metadata"]["signatureDef"]
signature_key = list(signature_def.keys())[0]
prediction_request = predict_pb2.PredictRequest()
prediction_request.model_spec.name = "default"
prediction_request.model_spec.signature_name = signature_key
for column_name, value in transformed_sample.items():
column_type = ctx.get_inferred_column_type(column_name)
data_type = tf_lib.CORTEX_TYPE_TO_TF_TYPE[column_type]
shape = [1]
if util.is_list(value):
shape = [len(value)]
tensor_proto = tf.make_tensor_proto([value],
dtype=data_type,
shape=shape)
prediction_request.inputs[column_name].CopyFrom(tensor_proto)
return prediction_request
def create_transformer_inputs_from_map(input, col_value_map):
if util.is_str(input):
if util.is_resource_ref(input):
res_name = util.get_resource_ref(input)
return col_value_map[res_name]
return input
if util.is_list(input):
replaced = []
for item in input:
replaced.append(create_transformer_inputs_from_map(item, col_value_map))
return replaced
if util.is_dict(input):
replaced = {}
for key, val in input.items():
key_replaced = create_transformer_inputs_from_map(key, col_value_map)
val_replaced = create_transformer_inputs_from_map(val, col_value_map)
replaced[key_replaced] = val_replaced
return replaced
return input
def populate_values(self, input, input_schema, preserve_column_refs):
if input is None:
if input_schema is None:
return None
if input_schema.get("_allow_null") == True:
return None
raise UserException("Null value is not allowed")
if util.is_resource_ref(input):
res_name = util.get_resource_ref(input)
if res_name in self.constants:
if self.constants[res_name].get("value") is not None:
const_val = self.constants[res_name]["value"]
elif self.constants[res_name].get("path") is not None:
const_val = self.storage.get_json_external(self.constants[res_name]["path"])
try:
return self.populate_values(const_val, input_schema, preserve_column_refs)
except CortexException as e:
e.wrap("constant " + res_name)
raise
if res_name in self.aggregates:
agg_val = self.get_obj(self.aggregates[res_name]["key"])
try:
return self.populate_values(agg_val, input_schema, preserve_column_refs)
except CortexException as e:
e.wrap("aggregate " + res_name)
raise
if res_name in self.columns:
if input_schema is not None:
col_type = self.get_inferred_column_type(res_name)
if col_type not in input_schema["_type"]:
raise UserException(
"column {}: unsupported input type (expected type {}, got type {})".format(
res_name,
util.data_type_str(input_schema["_type"]),
util.data_type_str(col_type),
)
)
if preserve_column_refs:
return input
else:
return res_name
if util.is_list(input):
elem_schema = None
if input_schema is not None:
if not util.is_list(input_schema["_type"]):
raise UserException(
"unsupported input type (expected type {}, got {})".format(
util.data_type_str(input_schema["_type"]), util.user_obj_str(input)
)
)
elem_schema = input_schema["_type"][0]
min_count = input_schema.get("_min_count")
if min_count is not None and len(input) < min_count:
raise UserException(
"list has length {}, but the minimum allowed length is {}".format(
len(input), min_count
)
)
max_count = input_schema.get("_max_count")
if max_count is not None and len(input) > max_count:
raise UserException(
"list has length {}, but the maximum allowed length is {}".format(
len(input), max_count
)
)
casted = []
for i, elem in enumerate(input):
try:
casted.append(self.populate_values(elem, elem_schema, preserve_column_refs))
except CortexException as e:
e.wrap("index " + i)
raise
return casted
if util.is_dict(input):
if input_schema is None:
casted = {}
for key, val in input.items():
key_casted = self.populate_values(key, None, preserve_column_refs)
try:
val_casted = self.populate_values(val, None, preserve_column_refs)
except CortexException as e:
e.wrap(util.user_obj_str(key))
raise
casted[key_casted] = val_casted
return casted
if not util.is_dict(input_schema["_type"]):
raise UserException(
"unsupported input type (expected type {}, got {})".format(
util.data_type_str(input_schema["_type"]), util.user_obj_str(input)
)
)
min_count = input_schema.get("_min_count")
if min_count is not None and len(input) < min_count:
raise UserException(
"map has length {}, but the minimum allowed length is {}".format(
len(input), min_count
)
)
max_count = input_schema.get("_max_count")
if max_count is not None and len(input) > max_count:
raise UserException(
"map has length {}, but the maximum allowed length is {}".format(
len(input), max_count
)
)
is_generic_map = False
if len(input_schema["_type"]) == 1:
input_type_key = next(iter(input_schema["_type"].keys()))
if is_compound_type(input_type_key):
is_generic_map = True
generic_map_key_schema = input_schema_from_type_schema(input_type_key)
generic_map_value = input_schema["_type"][input_type_key]
if is_generic_map:
casted = {}
for key, val in input.items():
key_casted = self.populate_values(
key, generic_map_key_schema, preserve_column_refs
)
try:
val_casted = self.populate_values(
val, generic_map_value, preserve_column_refs
)
except CortexException as e:
e.wrap(util.user_obj_str(key))
raise
casted[key_casted] = val_casted
return casted
# fixed map
casted = {}
for key, val_schema in input_schema["_type"].items():
if key in input:
val = input[key]
else:
if val_schema.get("_optional") is not True:
raise UserException("missing key: " + util.user_obj_str(key))
if val_schema.get("_default") is None:
continue
val = val_schema["_default"]
try:
val_casted = self.populate_values(val, val_schema, preserve_column_refs)
except CortexException as e:
e.wrap(util.user_obj_str(key))
raise
casted[key] = val_casted
return casted
if input_schema is None:
return input
if not util.is_str(input_schema["_type"]):
raise UserException(
"unsupported input type (expected type {}, got {})".format(
util.data_type_str(input_schema["_type"]), util.user_obj_str(input)
)
)
return cast_compound_type(input, input_schema["_type"])
def predict(app_name, api_name):
try:
payload = request.get_json()
except Exception as e:
return "Malformed JSON", status.HTTP_400_BAD_REQUEST
sess = local_cache["sess"]
api = local_cache["api"]
request_handler = local_cache.get("request_handler")
input_metadata = local_cache["input_metadata"]
output_metadata = local_cache["output_metadata"]
response = {}
if not util.is_dict(payload) or "samples" not in payload:
util.log_pretty_flat(payload, logging_func=logger.error)
return prediction_failed(
payload, "top level `samples` key not found in request")
predictions = []
samples = payload["samples"]
if not util.is_list(samples):
util.log_pretty_flat(samples, logging_func=logger.error)
return prediction_failed(
payload,
"expected the value of key `samples` to be a list of json objects")
for i, sample in enumerate(payload["samples"]):
try:
logger.info("sample: " + util.pp_str_flat(sample))
prepared_sample = sample
if request_handler is not None and util.has_function(
request_handler, "pre_inference"):
prepared_sample = request_handler.pre_inference(
sample, input_metadata)
logger.info("pre_inference: " +
util.pp_str_flat(prepared_sample))
inference_input = convert_to_onnx_input(prepared_sample,
input_metadata)
model_outputs = sess.run([], inference_input)
result = []
for model_output in model_outputs:
if type(model_output) is np.ndarray:
result.append(model_output.tolist())
else:
result.append(model_output)
logger.info("inference: " + util.pp_str_flat(result))
if request_handler is not None and util.has_function(
request_handler, "post_inference"):
result = request_handler.post_inference(
result, output_metadata)
logger.info("post_inference: " + util.pp_str_flat(result))
prediction = {"prediction": result}
except CortexException as e:
e.wrap("error", "sample {}".format(i + 1))
logger.error(str(e))
logger.exception(
"An error occurred, see `cx logs -v api {}` for more details.".
format(api["name"]))
return prediction_failed(sample, str(e))
except Exception as e:
logger.exception(
"An error occurred, see `cx logs -v api {}` for more details.".
format(api["name"]))
return prediction_failed(sample, str(e))
predictions.append(prediction)
response["predictions"] = predictions
response["resource_id"] = api["id"]
return jsonify(response)
def predict(deployment_name, api_name):
try:
payload = request.get_json()
except Exception as e:
return "Malformed JSON", status.HTTP_400_BAD_REQUEST
ctx = local_cache["ctx"]
api = local_cache["api"]
response = {}
if not util.is_dict(payload) or "samples" not in payload:
util.log_pretty_flat(payload, logging_func=logger.error)
return prediction_failed(
payload, "top level `samples` key not found in request")
logger.info("Predicting " +
util.pluralize(len(payload["samples"]), "sample", "samples"))
predictions = []
samples = payload["samples"]
if not util.is_list(samples):
util.log_pretty_flat(samples, logging_func=logger.error)
return prediction_failed(
payload,
"expected the value of key `samples` to be a list of json objects")
for i, sample in enumerate(payload["samples"]):
util.log_indent("sample {}".format(i + 1), 2)
if util.is_resource_ref(api["model"]):
is_valid, reason = is_valid_sample(sample)
if not is_valid:
return prediction_failed(sample, reason)
for column in local_cache["required_inputs"]:
column_type = ctx.get_inferred_column_type(column["name"])
sample[column["name"]] = util.upcast(sample[column["name"]],
column_type)
try:
result = run_predict(sample)
except CortexException as e:
e.wrap("error", "sample {}".format(i + 1))
logger.error(str(e))
logger.exception(
"An error occurred, see `cortex logs -v api {}` for more details."
.format(api["name"]))
return prediction_failed(sample, str(e))
except Exception as e:
logger.exception(
"An error occurred, see `cortex logs -v api {}` for more details."
.format(api["name"]))
# Show signature def for external models (since we don't validate input)
schemaStr = ""
signature_def = local_cache["metadata"]["signatureDef"]
if (not util.is_resource_ref(api["model"]) and
signature_def.get("predict") is not None # Just to be safe
and signature_def["predict"].get("inputs") is
not None # Just to be safe
):
schemaStr = "\n\nExpected shema:\n" + util.pp_str(
signature_def["predict"]["inputs"])
return prediction_failed(sample, str(e) + schemaStr)
predictions.append(result)
response["predictions"] = predictions
response["resource_id"] = api["id"]
return jsonify(response)