def Predict(self, request, context): """ Predict -- provides access to loaded TensorFlow model. """ # check if model with was requested # is available on server with proper version model_name = request.model_spec.name requested_version = request.model_spec.version.value valid_model_spec, version = check_availability_of_requested_model( models=self.models, requested_version=requested_version, model_name=model_name) if not valid_model_spec: context.set_code(StatusCode.NOT_FOUND) context.set_details(WRONG_MODEL_METADATA.format(model_name, requested_version)) logger.debug("PREDICT, invalid model spec from request, {} - {}" .format(model_name, requested_version)) return predict_pb2.PredictResponse() start_time = datetime.datetime.now() occurred_problem, inference_input, batch_size, code = \ prepare_input_data(models=self.models, model_name=model_name, version=version, data=request.inputs) deserialization_end_time = datetime.datetime.now() duration = (deserialization_end_time - start_time)\ .total_seconds() * 1000 logger.debug("PREDICT; input deserialization completed; {}; {}; {}ms" .format(model_name, version, duration)) if occurred_problem: context.set_code(code) context.set_details(inference_input) logger.debug("PREDICT, problem with input data. Exit code {}" .format(code)) return predict_pb2.PredictResponse() inference_start_time = datetime.datetime.now() inference_output = self.models[model_name].engines[version] \ .infer(inference_input, batch_size) inference_end_time = datetime.datetime.now() duration = (inference_end_time - inference_start_time)\ .total_seconds() * 1000 logger.debug("PREDICT; inference execution completed; {}; {}; {}ms" .format(model_name, version, duration)) response = prepare_output_as_list(inference_output=inference_output, model_available_outputs=self.models [model_name].engines[version]. model_keys['outputs']) response.model_spec.name = model_name response.model_spec.version.value = version response.model_spec.signature_name = SIGNATURE_NAME serialization_end_time = datetime.datetime.now() duration = (serialization_end_time - inference_end_time)\ .total_seconds() * 1000 logger.debug("PREDICT; inference results serialization completed;" " {}; {}; {}ms".format(model_name, version, duration)) return response
def Predict(self, request, context): """ Predict -- provides access to loaded TensorFlow model. """ # check if requested model # is available on server with proper version model_name = request.model_spec.name requested_version = request.model_spec.version.value valid_model_spec, version = check_availability_of_requested_model( models=self.models, requested_version=requested_version, model_name=model_name) if not valid_model_spec: context.set_code(StatusCode.NOT_FOUND) context.set_details( WRONG_MODEL_SPEC.format(model_name, requested_version)) logger.debug( "PREDICT, invalid model spec from request, {} - {}".format( model_name, requested_version)) return predict_pb2.PredictResponse() target_engine = self.models[model_name].engines[version] start_time = datetime.datetime.now() inference_input, error_message = \ prepare_input_data(target_engine=target_engine, data=request.inputs, service_type=GRPC) deserialization_end_time = datetime.datetime.now() duration = \ (deserialization_end_time - start_time).total_seconds() * 1000 logger.debug( "PREDICT; input deserialization completed; {}; {}; {}ms".format( model_name, version, duration)) if error_message is not None: code = statusCodes['invalid_arg'][GRPC] context.set_code(code) context.set_details(error_message) logger.debug( "PREDICT, problem with input data. Exit code {}".format(code)) return predict_pb2.PredictResponse() target_engine = self.models[model_name].engines[version] target_engine.in_use.acquire() ################################################ # Reshape network inputs if needed reshape_param = target_engine.detect_shapes_incompatibility( inference_input) if reshape_param is not None: error_message = target_engine.reshape(reshape_param) if error_message is not None: code = statusCodes['invalid_arg'][GRPC] context.set_code(code) context.set_details(error_message) target_engine.in_use.release() return predict_pb2.PredictResponse() ################################################ inference_start_time = datetime.datetime.now() inference_output, error_message = target_engine.infer(inference_input) if error_message is not None: code = statusCodes['invalid_arg'][GRPC] context.set_code(code) context.set_details(error_message) target_engine.in_use.release() return predict_pb2.PredictResponse() inference_end_time = datetime.datetime.now() target_engine.in_use.release() duration = \ (inference_end_time - inference_start_time).total_seconds() * 1000 logger.debug( "PREDICT; inference execution completed; {}; {}; {}ms".format( model_name, version, duration)) response = prepare_output_as_list( inference_output=inference_output, model_available_outputs=target_engine.model_keys['outputs']) response.model_spec.name = model_name response.model_spec.version.value = version response.model_spec.signature_name = SIGNATURE_NAME serialization_end_time = datetime.datetime.now() duration = \ (serialization_end_time - inference_end_time).total_seconds() * 1000 logger.debug("PREDICT; inference results serialization completed;" " {}; {}; {}ms".format(model_name, version, duration)) return response
def Predict(self, request, context): """ Predict -- provides access to loaded TensorFlow model. """ # check if requested model # is available on server with proper version model_name = request.model_spec.name requested_version = request.model_spec.version.value valid_model_spec, version = check_availability_of_requested_model( models=self.models, requested_version=requested_version, model_name=model_name) if not valid_model_spec: context.set_code(StatusCode.NOT_FOUND) context.set_details(WRONG_MODEL_SPEC.format(model_name, requested_version)) logger.debug("PREDICT, invalid model spec from request, {} - {}" .format(model_name, requested_version)) return predict_pb2.PredictResponse() target_engine = self.models[model_name].engines[version] deserialization_start_time = datetime.datetime.now() inference_input, error_message = \ prepare_input_data(target_engine=target_engine, data=request.inputs, service_type=GRPC) duration = (datetime.datetime.now() - deserialization_start_time).total_seconds() * 1000 logger.debug("PREDICT; input deserialization completed; {}; {}; {} ms" .format(model_name, version, duration)) if error_message is not None: code = statusCodes['invalid_arg'][GRPC] context.set_code(code) context.set_details(error_message) logger.debug("PREDICT, problem with input data. Exit code {}" .format(code)) return predict_pb2.PredictResponse() target_engine = self.models[model_name].engines[version] inference_request = Request(inference_input) target_engine.requests_queue.put(inference_request) inference_output, used_ireq_index = inference_request.wait_for_result() if type(inference_output) is str: code = statusCodes['invalid_arg'][GRPC] context.set_code(code) context.set_details(inference_output) logger.debug("PREDICT, problem during inference execution. Exit " "code {}".format(code)) target_engine.free_ireq_index_queue.put(used_ireq_index) return predict_pb2.PredictResponse() serialization_start_time = datetime.datetime.now() response = prepare_output_as_list( inference_output=inference_output, model_available_outputs=target_engine.model_keys['outputs']) response.model_spec.name = model_name response.model_spec.version.value = version response.model_spec.signature_name = SIGNATURE_NAME duration = (datetime.datetime.now() - serialization_start_time).total_seconds() * 1000 logger.debug("PREDICT; inference results serialization completed;" " {}; {}; {} ms".format(model_name, version, duration)) target_engine.free_ireq_index_queue.put(used_ireq_index) return response