def test_server_start():
port = random.randint(1000, 65535)
serving_config = ServingConfig(http_port=port)
python_config = PythonConfig(
python_code="first += 2",
python_inputs={"first": "NDARRAY"},
python_outputs={"first": "NDARRAY"},
)
step = PythonStep().step(python_config)
server = Server(steps=step, serving_config=serving_config)
server.start()
client = server.get_client()
data_input = {"default": np.load("../data/input-0.npy")}
assert is_port_in_use(port)
try:
predicted = client.predict(data_input)
print(predicted)
server.stop()
except Exception as e:
print(e)
server.stop()
def test_setup_and_run_start():
port = random.randint(1000, 65535)
serving_config = ServingConfig(http_port=port)
python_config = PythonConfig(
python_code=
"def setup(): pass\ndef run(input): {'output': np.array(input + 2)}",
python_inputs={"input": "NDARRAY"},
python_outputs={"output": "NDARRAY"},
setup_and_run=True,
)
step = PythonStep().step(python_config)
server = Server(steps=step, serving_config=serving_config)
server.start()
client = server.get_client()
data_input = {"default": np.asarray([42.0, 1.0])}
assert is_port_in_use(port)
try:
predicted = client.predict(data_input)
print(predicted)
server.stop()
except Exception as e:
print(e)
server.stop()
def test_python_script_prediction():
port = 1337
serving_config = ServingConfig(http_port=port)
work_dir = os.path.abspath(".")
python_config = PythonConfig(
python_path=default_python_path(work_dir),
python_code_path=os.path.join(work_dir, "simple.py"),
python_inputs={"first": "NDARRAY"},
python_outputs={"second": "NDARRAY"},
)
step = PythonStep().step(python_config)
server = Server(steps=step, serving_config=serving_config)
server.start()
client = Client(port=port)
if is_port_in_use(port):
input_array = np.load("../data/input-0.npy")
predicted = client.predict(input_array)
print(predicted)
server.stop()
else:
server.stop()
raise Exception("Server not running on specified port")
def test_server_start():
input_names = [
"IteratorGetNext:0", "IteratorGetNext:1", "IteratorGetNext:4"
]
output_names = ["loss/Softmax"]
port = random.randint(1000, 65535)
parallel_inference_config = ParallelInferenceConfig(workers=1)
serving_config = ServingConfig(
http_port=port,
input_data_type='NUMPY',
output_data_type='NUMPY',
log_timings=True,
parallel_inference_config=parallel_inference_config)
tensorflow_config = TensorFlowConfig(
model_config_type=ModelConfigType(
model_type='TENSORFLOW', model_loading_path='bert_mrpc_frozen.pb'),
tensor_data_types_config=TensorDataTypesConfig(
input_data_types={
'IteratorGetNext:0': 'INT32',
'IteratorGetNext:1': 'INT32',
'IteratorGetNext:4': 'INT32'
}))
model_pipeline_step = ModelPipelineStep(model_config=tensorflow_config,
serving_config=serving_config,
input_names=input_names,
output_names=output_names)
inference = InferenceConfiguration(serving_config=serving_config,
pipeline_steps=[model_pipeline_step])
server = Server(config=inference,
extra_start_args='-Xmx8g',
jar_path='konduit.jar')
server.start()
client = Client(input_names=input_names,
output_names=output_names,
input_type='NUMPY',
endpoint_output_type='NUMPY',
url='http://localhost:' + str(port))
data_input = {
'IteratorGetNext:0': np.load('../data/input-0.npy'),
'IteratorGetNext:1': np.load('../data/input-1.npy'),
'IteratorGetNext:4': np.load('../data/input-4.npy')
}
print('Process started. Sleeping 30 seconds.')
time.sleep(30)
assert is_port_in_use(port)
try:
predicted = client.predict(data_input)
print(predicted)
server.stop()
except Exception as e:
print(e)
server.stop()
def test_server_start():
input_names = ['default']
output_names = ['default']
port = random.randint(1000, 65535)
parallel_inference_config = ParallelInferenceConfig(workers=1)
serving_config = ServingConfig(
http_port=port,
input_data_type='NUMPY',
output_data_type='NUMPY',
log_timings=True,
parallel_inference_config=parallel_inference_config)
python_config = PythonConfig(
python_code='first += 2',
python_inputs={'first': 'NDARRAY'},
python_outputs={'first': 'NDARRAY'},
)
python_pipeline_step = PythonPipelineStep(
input_names=input_names,
output_names=output_names,
input_schemas=({
'default': ['NDArray']
}),
output_schemas=({
'default': ['NDArray']
}),
input_column_names={'default': ['first']},
output_column_names={'default': ['first']},
python_configs={'default': python_config})
inference = InferenceConfiguration(serving_config=serving_config,
pipeline_steps=[python_pipeline_step])
server = Server(config=inference,
extra_start_args='-Xmx8g',
jar_path='konduit.jar')
server.start()
print('Process started. Sleeping 10 seconds.')
client = Client(input_names=input_names,
output_names=output_names,
input_type='NUMPY',
endpoint_output_type='NUMPY',
url='http://localhost:' + str(port))
data_input = {
'default': np.load('../data/input-0.npy'),
}
time.sleep(4)
assert is_port_in_use(port)
try:
predicted = client.predict(data_input)
print(predicted)
server.stop()
except Exception as e:
print(e)
server.stop()
def test_server_start():
input_names = [
"IteratorGetNext:0", "IteratorGetNext:1", "IteratorGetNext:4"
]
output_names = ["loss/Softmax"]
port = random.randint(1000, 65535)
parallel_inference_config = ParallelInferenceConfig(workers=1)
serving_config = ServingConfig(
http_port=port,
input_data_format="NUMPY",
output_data_format="NUMPY",
log_timings=True,
)
tensorflow_config = TensorFlowConfig(
model_config_type=ModelConfigType(
model_type="TENSORFLOW", model_loading_path="bert_mrpc_frozen.pb"),
tensor_data_types_config=TensorDataTypesConfig(
input_data_types={
"IteratorGetNext:0": "INT32",
"IteratorGetNext:1": "INT32",
"IteratorGetNext:4": "INT32",
}),
)
model_pipeline_step = ModelStep(
model_config=tensorflow_config,
parallel_inference_config=parallel_inference_config,
input_names=input_names,
output_names=output_names,
)
inference = InferenceConfiguration(serving_config=serving_config,
steps=[model_pipeline_step])
server = Server(inference_config=inference,
extra_start_args="-Xmx8g",
jar_path="konduit.jar")
server.start()
client = Client(input_data_format="NUMPY",
prediction_type="NUMPY",
port=port)
data_input = {
"IteratorGetNext:0": np.load("../data/input-0.npy"),
"IteratorGetNext:1": np.load("../data/input-1.npy"),
"IteratorGetNext:4": np.load("../data/input-4.npy"),
}
assert is_port_in_use(port)
try:
predicted = client.predict(data_input)
print(predicted)
server.stop()
except Exception as e:
print(e)
server.stop()
def test_build_tp(output_format):
schema = pydatavec.Schema()
schema.add_string_column("first")
tp = pydatavec.TransformProcess(schema)
tp.append_string("first", "two")
java_tp = tp.to_java()
tp_json = java_tp.toJson()
load_java_tp(tp_json)
_ = json.dumps(tp_json)
as_python_json = json.loads(tp_json)
transform_process = (TransformProcessStep().set_input(
column_names=["first"],
types=["String"
]).set_output(column_names=["first"],
types=["String"
]).transform_process(as_python_json))
port = random.randint(1000, 65535)
serving_config = ServingConfig(
http_port=port,
input_data_format="JSON",
output_data_format=output_format,
log_timings=True,
)
inference_config = InferenceConfiguration(serving_config=serving_config,
steps=[transform_process])
as_json = config_to_dict_with_type(inference_config)
inference_from_json(as_json)
server = Server(
inference_config=inference_config,
extra_start_args="-Xmx8g",
jar_path="konduit.jar",
)
server.start()
client = server.get_client()
assert is_port_in_use(port)
try:
predicted = client.predict({"first": "value"})
print(predicted)
server.stop()
except Exception as e:
print(e)
server.stop()
def test_python_script_prediction():
work_dir = os.path.abspath(".")
python_config = PythonConfig(
python_path=default_python_path(work_dir),
python_code_path=os.path.join(work_dir, "simple.py"),
python_inputs={"first": "NDARRAY"},
python_outputs={"second": "NDARRAY"},
)
step = PythonStep().step(python_config)
server = Server(steps=step, serving_config=ServingConfig())
_, port, started = server.start()
assert started
assert is_port_in_use(port)
client = Client(port=port)
try:
input_array = np.load("../data/input-0.npy")
predicted = client.predict(input_array)
print(predicted)
server.stop()
except Exception as e:
print(e)
server.stop()
def test_server_start():
port = random.randint(1000, 65535)
parallel_inference_config = ParallelInferenceConfig(workers=1)
serving_config = ServingConfig(
http_port=port,
input_data_type='NUMPY',
output_data_type='NUMPY',
log_timings=True,
parallel_inference_config=parallel_inference_config)
tensorflow_config = TensorFlowConfig(
model_config_type=ModelConfigType(
model_type='TENSORFLOW',
model_loading_path='./bert_mrpc_frozen.pb'),
tensor_data_types_config=TensorDataTypesConfig(
input_data_types={
'IteratorGetNext:0': 'INT32',
'IteratorGetNext:1': 'INT32',
'IteratorGetNext:4': 'INT32'
}))
model_pipeline_step = ModelPipelineStep(model_config=tensorflow_config,
serving_config=serving_config,
input_names=[
"IteratorGetNext:0",
"IteratorGetNext:1",
"IteratorGetNext:4"
],
output_names=["loss/Softmax"])
inference_config = InferenceConfiguration(
serving_config=serving_config, pipeline_steps=[model_pipeline_step])
server = Server(config=inference_config,
extra_start_args='-Xmx8g',
jar_path='konduit.jar')
server.start()
print('Process started. Sleeping 30 seconds.')
time.sleep(30)
assert is_port_in_use(port)
print('Done sleeping. Assuming server alive. Killing process.')
server.stop()
def test_server_start():
server_id = "tensorflow_server"
input_names = [
"IteratorGetNext:0", "IteratorGetNext:1", "IteratorGetNext:4"
]
output_names = ["loss/Softmax"]
parallel_inference_config = ParallelInferenceConfig(workers=1)
serving_config = ServingConfig(
output_data_format="NUMPY",
log_timings=True,
)
model_pipeline_step = TensorFlowStep(
path="bert_mrpc_frozen.pb",
input_data_types={
input_names[0]: "INT32",
input_names[1]: "INT32",
input_names[2]: "INT32",
},
parallel_inference_config=parallel_inference_config,
input_names=input_names,
output_names=output_names,
)
inference = InferenceConfiguration(serving_config=serving_config,
steps=[model_pipeline_step])
server = Server(inference_config=inference, extra_start_args="-Xmx8g")
_, port, started = server.start(server_id)
data_input = {
input_names[0]: np.load("../data/input-0.npy"),
input_names[1]: np.load("../data/input-1.npy"),
input_names[2]: np.load("../data/input-4.npy"),
}
assert started # will be true if the server was started
assert is_port_in_use(port)
client = Client(input_data_format="NUMPY",
prediction_type="RAW",
port=port)
try:
predicted = client.predict(data_input)
print(predicted)
except Exception as e:
print(e)
finally:
server.stop(server_id)
python_code_path=os.path.join(work_dir, "detect_image.py"),
python_inputs={"image": "NDARRAY"},
python_outputs={"num_boxes": "NDARRAY"},
)
# Configure a Python pipeline step for your Python code. Internally, konduit will take numpy arrays as input and
# return data in JSON format. Note that the Python variable 'image' and the Konduit step name 'image_data' are separate
# things here.
step_input_name = "image_data"
python_pipeline_step = PythonStep().step(python_config,
input_name=step_input_name)
serving_config = ServingConfig(http_port=1337, output_data_format="JSON")
# Start a konduit server and wait for it to start
server = Server(serving_config=serving_config, steps=[python_pipeline_step])
server.start()
client = Client(port="1337", timeout=30)
encoded_image = cv2.cvtColor(
cv2.imread("./Ultra-Light-Fast-Generic-Face-Detector-1MB/imgs/1.jpg"),
cv2.COLOR_BGR2RGB,
)
encoded_image = encoded_image.astype("int16")
print(">>> Image input shape")
print(encoded_image.shape)
try:
predicted = client.predict({step_input_name: encoded_image})
assert predicted[step_input_name]["ndArray"]["data"][0] == 51
finally:
def test_build_tp():
TransformProcessBuilder = autoclass(
'org.datavec.api.transform.TransformProcess$Builder')
TransformProcess = autoclass('org.datavec.api.transform.TransformProcess')
SchemaBuilder = autoclass(
'org.datavec.api.transform.schema.Schema$Builder')
schema = SchemaBuilder().addColumnString('first').build()
tp = TransformProcessBuilder(schema).appendStringColumnTransform(
"first", "two").build()
tp_json = tp.toJson()
from_json = TransformProcess.fromJson(tp_json)
json_tp = json.dumps(tp_json)
as_python_json = json.loads(tp_json)
transform_process = TransformProcessPipelineStep(
transform_processes={'default': as_python_json},
input_names=['default'],
output_names=['default'],
input_schemas={'default': ['String']},
output_schemas={'default': ['String']},
input_column_names={'default': ['first']},
output_column_names={'default': ['first']})
input_names = ['default']
output_names = ['default']
port = random.randint(1000, 65535)
parallel_inference_config = ParallelInferenceConfig(workers=1)
serving_config = ServingConfig(
http_port=port,
input_data_type='JSON',
output_data_type='JSON',
log_timings=True,
parallel_inference_config=parallel_inference_config)
inference = InferenceConfiguration(serving_config=serving_config,
pipeline_steps=[transform_process])
as_json = json_with_type(inference)
InferenceConfigurationJava = autoclass(
'ai.konduit.serving.InferenceConfiguration')
config = InferenceConfigurationJava.fromJson(json.dumps(as_json))
server = Server(config=inference,
extra_start_args='-Xmx8g',
jar_path='konduit.jar')
server.start()
print('Process started. Sleeping 10 seconds.')
client = Client(input_names=input_names,
output_names=output_names,
return_output_type='JSON',
input_type='JSON',
endpoint_output_type='RAW',
url='http://localhost:' + str(port))
data_input = {'first': 'value'}
time.sleep(30)
assert is_port_in_use(port)
try:
predicted = client.predict(data_input)
print(predicted)
server.stop()
except Exception as e:
print(e)
server.stop()
print('Running on port ' + str(port))
parallel_inference_config = ParallelInferenceConfig(workers=1)
serving_config = ServingConfig(
http_port=port,
input_data_type='JSON',
output_data_type='ARROW',
log_timings=True,
parallel_inference_config=parallel_inference_config)
inference = InferenceConfiguration(serving_config=serving_config,
pipeline_steps=[transform_process])
as_json = json_with_type(inference)
server = Server(config=inference,
extra_start_args='-Xmx8g',
jar_path=konduit_jar)
process = server.start()
print('Process started. Sleeping 10 seconds.')
client = Client(input_names=input_names,
output_names=output_names,
return_output_type='ARROW',
input_type='JSON',
endpoint_output_type='RAW',
url='http://localhost:' + str(port))
data_input = {'first': 'value'}
try:
predicted = client.predict(data_input)
print(predicted)
process.wait()
server.stop()