def test_python_script_prediction():
work_dir = os.path.abspath(".")
python_config = PythonConfig(
python_path=default_python_path(work_dir),
python_code_path=os.path.join(work_dir, "simple.py"),
python_inputs={"first": "NDARRAY"},
python_outputs={"second": "NDARRAY"},
)
step = PythonStep().step(python_config)
server = Server(steps=step, serving_config=ServingConfig())
_, port, started = server.start()
assert started
assert is_port_in_use(port)
client = Client(port=port)
try:
input_array = np.load("../data/input-0.npy")
predicted = client.predict(input_array)
print(predicted)
server.stop()
except Exception as e:
print(e)
server.stop()
def test_python_script_prediction():
port = 1337
serving_config = ServingConfig(http_port=port)
work_dir = os.path.abspath(".")
python_config = PythonConfig(
python_path=default_python_path(work_dir),
python_code_path=os.path.join(work_dir, "simple.py"),
python_inputs={"first": "NDARRAY"},
python_outputs={"second": "NDARRAY"},
)
step = PythonStep().step(python_config)
server = Server(steps=step, serving_config=serving_config)
server.start()
client = Client(port=port)
if is_port_in_use(port):
input_array = np.load("../data/input-0.npy")
predicted = client.predict(input_array)
print(predicted)
server.stop()
else:
server.stop()
raise Exception("Server not running on specified port")
def get_client(self, output_data_format=None):
"""Get a Konduit Client instance from this Server instance.
:param output_data_format: optional, same as in Client signature
:return: konduit.Client
"""
serving_config = self.config._get_serving_config()
steps = self.config._get_steps()
input_names = []
output_names = []
for step in steps:
input_names += step._get_input_names()
output_names += step._get_output_names()
port = serving_config._get_http_port()
host = serving_config._get_listen_host()
if not host.startswith("http://"):
host = "http://" + host
input_data_format = serving_config._get_input_data_format()
prediction_type = serving_config._get_prediction_type()
if not output_data_format:
output_data_format = serving_config._get_output_data_format()
return Client(
host=host,
port=port,
input_data_format=input_data_format,
output_data_format=output_data_format,
prediction_type=prediction_type,
input_names=input_names,
output_names=output_names,
)
def test_server_start():
input_names = [
"IteratorGetNext:0", "IteratorGetNext:1", "IteratorGetNext:4"
]
output_names = ["loss/Softmax"]
port = random.randint(1000, 65535)
parallel_inference_config = ParallelInferenceConfig(workers=1)
serving_config = ServingConfig(
http_port=port,
input_data_type='NUMPY',
output_data_type='NUMPY',
log_timings=True,
parallel_inference_config=parallel_inference_config)
tensorflow_config = TensorFlowConfig(
model_config_type=ModelConfigType(
model_type='TENSORFLOW', model_loading_path='bert_mrpc_frozen.pb'),
tensor_data_types_config=TensorDataTypesConfig(
input_data_types={
'IteratorGetNext:0': 'INT32',
'IteratorGetNext:1': 'INT32',
'IteratorGetNext:4': 'INT32'
}))
model_pipeline_step = ModelPipelineStep(model_config=tensorflow_config,
serving_config=serving_config,
input_names=input_names,
output_names=output_names)
inference = InferenceConfiguration(serving_config=serving_config,
pipeline_steps=[model_pipeline_step])
server = Server(config=inference,
extra_start_args='-Xmx8g',
jar_path='konduit.jar')
server.start()
client = Client(input_names=input_names,
output_names=output_names,
input_type='NUMPY',
endpoint_output_type='NUMPY',
url='http://localhost:' + str(port))
data_input = {
'IteratorGetNext:0': np.load('../data/input-0.npy'),
'IteratorGetNext:1': np.load('../data/input-1.npy'),
'IteratorGetNext:4': np.load('../data/input-4.npy')
}
print('Process started. Sleeping 30 seconds.')
time.sleep(30)
assert is_port_in_use(port)
try:
predicted = client.predict(data_input)
print(predicted)
server.stop()
except Exception as e:
print(e)
server.stop()
def test_server_start():
input_names = ['default']
output_names = ['default']
port = random.randint(1000, 65535)
parallel_inference_config = ParallelInferenceConfig(workers=1)
serving_config = ServingConfig(
http_port=port,
input_data_type='NUMPY',
output_data_type='NUMPY',
log_timings=True,
parallel_inference_config=parallel_inference_config)
python_config = PythonConfig(
python_code='first += 2',
python_inputs={'first': 'NDARRAY'},
python_outputs={'first': 'NDARRAY'},
)
python_pipeline_step = PythonPipelineStep(
input_names=input_names,
output_names=output_names,
input_schemas=({
'default': ['NDArray']
}),
output_schemas=({
'default': ['NDArray']
}),
input_column_names={'default': ['first']},
output_column_names={'default': ['first']},
python_configs={'default': python_config})
inference = InferenceConfiguration(serving_config=serving_config,
pipeline_steps=[python_pipeline_step])
server = Server(config=inference,
extra_start_args='-Xmx8g',
jar_path='konduit.jar')
server.start()
print('Process started. Sleeping 10 seconds.')
client = Client(input_names=input_names,
output_names=output_names,
input_type='NUMPY',
endpoint_output_type='NUMPY',
url='http://localhost:' + str(port))
data_input = {
'default': np.load('../data/input-0.npy'),
}
time.sleep(4)
assert is_port_in_use(port)
try:
predicted = client.predict(data_input)
print(predicted)
server.stop()
except Exception as e:
print(e)
server.stop()
def test_server_start():
input_names = [
"IteratorGetNext:0", "IteratorGetNext:1", "IteratorGetNext:4"
]
output_names = ["loss/Softmax"]
port = random.randint(1000, 65535)
parallel_inference_config = ParallelInferenceConfig(workers=1)
serving_config = ServingConfig(
http_port=port,
input_data_format="NUMPY",
output_data_format="NUMPY",
log_timings=True,
)
tensorflow_config = TensorFlowConfig(
model_config_type=ModelConfigType(
model_type="TENSORFLOW", model_loading_path="bert_mrpc_frozen.pb"),
tensor_data_types_config=TensorDataTypesConfig(
input_data_types={
"IteratorGetNext:0": "INT32",
"IteratorGetNext:1": "INT32",
"IteratorGetNext:4": "INT32",
}),
)
model_pipeline_step = ModelStep(
model_config=tensorflow_config,
parallel_inference_config=parallel_inference_config,
input_names=input_names,
output_names=output_names,
)
inference = InferenceConfiguration(serving_config=serving_config,
steps=[model_pipeline_step])
server = Server(inference_config=inference,
extra_start_args="-Xmx8g",
jar_path="konduit.jar")
server.start()
client = Client(input_data_format="NUMPY",
prediction_type="NUMPY",
port=port)
data_input = {
"IteratorGetNext:0": np.load("../data/input-0.npy"),
"IteratorGetNext:1": np.load("../data/input-1.npy"),
"IteratorGetNext:4": np.load("../data/input-4.npy"),
}
assert is_port_in_use(port)
try:
predicted = client.predict(data_input)
print(predicted)
server.stop()
except Exception as e:
print(e)
server.stop()
def test_server_start():
server_id = "tensorflow_server"
input_names = [
"IteratorGetNext:0", "IteratorGetNext:1", "IteratorGetNext:4"
]
output_names = ["loss/Softmax"]
parallel_inference_config = ParallelInferenceConfig(workers=1)
serving_config = ServingConfig(
output_data_format="NUMPY",
log_timings=True,
)
model_pipeline_step = TensorFlowStep(
path="bert_mrpc_frozen.pb",
input_data_types={
input_names[0]: "INT32",
input_names[1]: "INT32",
input_names[2]: "INT32",
},
parallel_inference_config=parallel_inference_config,
input_names=input_names,
output_names=output_names,
)
inference = InferenceConfiguration(serving_config=serving_config,
steps=[model_pipeline_step])
server = Server(inference_config=inference, extra_start_args="-Xmx8g")
_, port, started = server.start(server_id)
data_input = {
input_names[0]: np.load("../data/input-0.npy"),
input_names[1]: np.load("../data/input-1.npy"),
input_names[2]: np.load("../data/input-4.npy"),
}
assert started # will be true if the server was started
assert is_port_in_use(port)
client = Client(input_data_format="NUMPY",
prediction_type="RAW",
port=port)
try:
predicted = client.predict(data_input)
print(predicted)
except Exception as e:
print(e)
finally:
server.stop(server_id)
def predict_numpy(host, port, numpy_data, input_names):
"""Get predictions for your pipeline from numpy data and input names"""
from konduit.client import Client
numpy_files = numpy_data.split(",")
input_names = input_names.split(",")
assert len(numpy_files) == len(input_names)
client = Client(host="http://" + host, port=port)
input_dict = {}
for i in range(len(numpy_files)):
input_dict[input_names[i]] = np.load(numpy_files[i])
print(client.predict(input_dict))
def test_model():
request_count = 10
client = Client(input_data_format="IMAGE", output_data_format="NUMPY", input_names=["default"], output_names=["output"], host="http://localhost", port=1337, prediction_type="NUMPY")
#client = client_from_file("konduit.yml")
with open("1902_airplane.png", "rb") as binary_file:
data = binary_file.read()
responses = []
start = time.time()
for _ in range(request_count):
responses.append(client.predict({"default": data}))
end = time.time()
print(responses[0])
print("%f seconds elapsed for %d requests (%f RPS)" % (end - start, request_count, (request_count / (end - start))))
def test_multipart_encode():
input_names = ["IteratorGetNext:0", "IteratorGetNext:1", "IteratorGetNext:4"]
output_names = ["loss/Softmax"]
port = random.randint(1000, 65535)
client = Client(
input_names=input_names,
output_names=output_names,
input_data_format="NUMPY",
output_data_format="NUMPY",
port=port,
)
input_data = {
"input1": Client._convert_numpy_to_binary(np.ones(1)),
"input2": Client._convert_numpy_to_binary(np.ones(2)),
}
converted = Client._convert_multi_part_inputs(input_data)
body, content_type = Client._encode_multi_part_input(converted)
client._convert_multi_part_output(content=body, content_type=content_type)
def test_multipart_encode():
input_names = [
"IteratorGetNext:0", "IteratorGetNext:1", "IteratorGetNext:4"
]
output_names = ["loss/Softmax"]
port = random.randint(1000, 65535)
client = Client(input_names=input_names,
output_names=output_names,
input_type='NUMPY',
endpoint_output_type='NUMPY',
url='http://localhost:' + str(port))
input = {
'input1': Client._convert_numpy_to_binary(np.ones(1)),
'input2': Client._convert_numpy_to_binary(np.ones(2))
}
converted = Client._convert_multi_part_inputs(input)
body, content_type = Client._encode_multi_part_input(converted)
output = client._convert_multi_part_output(content=body,
content_type=content_type)
print(output)
# Configure a Python pipeline step for your Python code. Internally, konduit will take Strings as input and output
# for this example.
python_pipeline_step = PythonStep().step(python_config)
serving_config = ServingConfig(http_port=1337,
input_data_format="JSON",
output_data_format="JSON")
# Start a konduit server and wait for it to start
server = Server(serving_config=serving_config, steps=[python_pipeline_step])
server.start()
# Initialize a konduit client that takes in and outputs JSON
client = Client(
input_data_format="JSON",
prediction_type="RAW",
output_data_format="JSON",
host="http://localhost",
port=1337,
)
# encode the image from a file to base64 and get back a prediction from the konduit server
encoded_image = to_base_64(
os.path.abspath("./Ultra-Light-Fast-Generic-Face-Detector-1MB/imgs/1.jpg"))
predicted = client.predict({"image": encoded_image})
# the actual output can be found under "num_boxes"
print(predicted)
assert predicted["num_boxes"] == "51"
server.stop()
def test_build_tp():
TransformProcessBuilder = autoclass(
'org.datavec.api.transform.TransformProcess$Builder')
TransformProcess = autoclass('org.datavec.api.transform.TransformProcess')
SchemaBuilder = autoclass(
'org.datavec.api.transform.schema.Schema$Builder')
schema = SchemaBuilder().addColumnString('first').build()
tp = TransformProcessBuilder(schema).appendStringColumnTransform(
"first", "two").build()
tp_json = tp.toJson()
from_json = TransformProcess.fromJson(tp_json)
json_tp = json.dumps(tp_json)
as_python_json = json.loads(tp_json)
transform_process = TransformProcessPipelineStep(
transform_processes={'default': as_python_json},
input_names=['default'],
output_names=['default'],
input_schemas={'default': ['String']},
output_schemas={'default': ['String']},
input_column_names={'default': ['first']},
output_column_names={'default': ['first']})
input_names = ['default']
output_names = ['default']
port = random.randint(1000, 65535)
parallel_inference_config = ParallelInferenceConfig(workers=1)
serving_config = ServingConfig(
http_port=port,
input_data_type='JSON',
output_data_type='JSON',
log_timings=True,
parallel_inference_config=parallel_inference_config)
inference = InferenceConfiguration(serving_config=serving_config,
pipeline_steps=[transform_process])
as_json = json_with_type(inference)
InferenceConfigurationJava = autoclass(
'ai.konduit.serving.InferenceConfiguration')
config = InferenceConfigurationJava.fromJson(json.dumps(as_json))
server = Server(config=inference,
extra_start_args='-Xmx8g',
jar_path='konduit.jar')
server.start()
print('Process started. Sleeping 10 seconds.')
client = Client(input_names=input_names,
output_names=output_names,
return_output_type='JSON',
input_type='JSON',
endpoint_output_type='RAW',
url='http://localhost:' + str(port))
data_input = {'first': 'value'}
time.sleep(30)
assert is_port_in_use(port)
try:
predicted = client.predict(data_input)
print(predicted)
server.stop()
except Exception as e:
print(e)
server.stop()
http_port=port,
input_data_type='JSON',
output_data_type='ARROW',
log_timings=True,
parallel_inference_config=parallel_inference_config)
inference = InferenceConfiguration(serving_config=serving_config,
pipeline_steps=[transform_process])
as_json = json_with_type(inference)
server = Server(config=inference,
extra_start_args='-Xmx8g',
jar_path=konduit_jar)
process = server.start()
print('Process started. Sleeping 10 seconds.')
client = Client(input_names=input_names,
output_names=output_names,
return_output_type='ARROW',
input_type='JSON',
endpoint_output_type='RAW',
url='http://localhost:' + str(port))
data_input = {'first': 'value'}
try:
predicted = client.predict(data_input)
print(predicted)
process.wait()
server.stop()
except Exception as e:
print(e)
server.stop()
