def process_request():
upstream = send_predict(query_frontend, "sleep_1", "")
if materialize:
upstream = ray.get(upstream)
downstream_1 = send_predict(query_frontend, "sleep_2", upstream)
downstream_2 = send_predict(query_frontend, "sleep_3", upstream)
ray.get([downstream_1, downstream_2])
def process_request(input_image):
upstream = send_predict(query_frontend, "preprocess", input_image)
middle_tier_results = [
send_predict(query_frontend, f"squeezenet_{i}", upstream)
for i in range(n_replicas)
]
downstream = send_predict(query_frontend, "ensemble", middle_tier_results)
ray.get([downstream])
def process_request():
input_image = np.random.randn(batchsize, 224, 224, 3)
log_with_timestamp("Request sent!")
upstream = send_predict(query_frontend, "preprocess", input_image)
middle_tier_results = [
send_predict(query_frontend, f"squeezenet_{i}", upstream)
for i in range(n_replicas)
]
downstream = send_predict(query_frontend, "ensemble", middle_tier_results)
log_with_timestamp(f"Result get! {ray.get([downstream])[0][:10]}")
async def predict():
log_with_timestamp("entering predict")
upstream = send_predict(query_frontend, "preprocess", input_image)
middle_tier_results = [
send_predict(query_frontend, f"squeezenet_{i}", upstream)
for i in range(n_replicas)
]
downstream = send_predict(query_frontend, "ensemble", middle_tier_results)
result = await async_api.as_future(downstream)
log_with_timestamp("exiting predict")
def test_pipeline(initialize_ray, query_frontend):
model_1 = "noop_1"
model_2 = "noop_2"
query_frontend.add_model.remote(model_1, NoopModelActor)
query_frontend.add_model.remote(model_2, NoopModelActor)
# test materialization
model_1_result = ray.get(send_predict(query_frontend, model_1, ""))
model_2_result = ray.get(
send_predict(query_frontend, model_2, model_1_result))
assert model_2_result == ""
# test no materialization
model_1_result = send_predict(query_frontend, model_1, "")
model_2_result = ray.get(
send_predict(query_frontend, model_2, model_1_result))
assert model_2_result == ""
def test_identity(initialize_ray, query_frontend):
model_name = "identity"
query_frontend.add_model.remote(model_name, IdentityModelActor)
result_oids = []
inputs = list(range(10))
for i in inputs:
result_oids.append(send_predict(query_frontend, model_name, i))
result = ray.get(result_oids)
assert result == inputs
def test_replicas(initialize_ray, query_frontend):
model_name = "sleep_replicated"
query_frontend.add_model.remote(model_name, SleepModelActor)
query_frontend.scale_up_model.remote(model_name, SleepModelActor, 3)
result_oids = []
inputs = list(range(30))
for i in inputs:
result_oids.append(send_predict(query_frontend, model_name, i))
ray.get(result_oids)
metrics = ray.get(query_frontend.get_metric.remote())
for (name, replica_id), metric in metrics.items():
if name == model_name:
assert len(metric["batch_size"]) != 0
def test_batching(initialize_ray, query_frontend):
# We use a sleep actor to test batching is the model queue has
# time to build up
model_name = "sleep"
query_frontend.add_model.remote(model_name, SleepModelActor)
result_oids = []
inputs = list(range(10))
for i in inputs:
result_oids.append(send_predict(query_frontend, model_name, i))
ray.get(result_oids)
metrics = ray.get(query_frontend.get_metric.remote())[("sleep",
0)]["batch_size"]
assert 2 in metrics
def test_sklearn(initialize_ray, query_frontend):
model_name = "linear_regression"
query_frontend.add_model.remote(model_name, SKLearnModelActor)
model = LinearRegression()
X = np.ones(shape=(100, 2))
y = np.ones(100)
model.fit(X, y)
result_oids = []
inputs = np.ones(shape=(100, 2)) * 2
for i in inputs:
result_oids.append(send_predict(query_frontend, model_name, i))
result = np.array(ray.get(result_oids)).flatten()
truth = model.predict(inputs).flatten()
assert np.array_equal(result, truth)
def process_request():
oids = []
for _ in range(num_replicas):
oids.append(send_predict(query_frontend, model_name, ""))
ray.get(oids)
def send_noop_and_wait():
ray.get(send_predict(query_frontend, model_name, ""))