def process_request(): upstream = send_predict(query_frontend, "sleep_1", "") if materialize: upstream = ray.get(upstream) downstream_1 = send_predict(query_frontend, "sleep_2", upstream) downstream_2 = send_predict(query_frontend, "sleep_3", upstream) ray.get([downstream_1, downstream_2])
def process_request(input_image): upstream = send_predict(query_frontend, "preprocess", input_image) middle_tier_results = [ send_predict(query_frontend, f"squeezenet_{i}", upstream) for i in range(n_replicas) ] downstream = send_predict(query_frontend, "ensemble", middle_tier_results) ray.get([downstream])
def process_request(): input_image = np.random.randn(batchsize, 224, 224, 3) log_with_timestamp("Request sent!") upstream = send_predict(query_frontend, "preprocess", input_image) middle_tier_results = [ send_predict(query_frontend, f"squeezenet_{i}", upstream) for i in range(n_replicas) ] downstream = send_predict(query_frontend, "ensemble", middle_tier_results) log_with_timestamp(f"Result get! {ray.get([downstream])[0][:10]}")
async def predict(): log_with_timestamp("entering predict") upstream = send_predict(query_frontend, "preprocess", input_image) middle_tier_results = [ send_predict(query_frontend, f"squeezenet_{i}", upstream) for i in range(n_replicas) ] downstream = send_predict(query_frontend, "ensemble", middle_tier_results) result = await async_api.as_future(downstream) log_with_timestamp("exiting predict")
def test_pipeline(initialize_ray, query_frontend): model_1 = "noop_1" model_2 = "noop_2" query_frontend.add_model.remote(model_1, NoopModelActor) query_frontend.add_model.remote(model_2, NoopModelActor) # test materialization model_1_result = ray.get(send_predict(query_frontend, model_1, "")) model_2_result = ray.get( send_predict(query_frontend, model_2, model_1_result)) assert model_2_result == "" # test no materialization model_1_result = send_predict(query_frontend, model_1, "") model_2_result = ray.get( send_predict(query_frontend, model_2, model_1_result)) assert model_2_result == ""
def test_identity(initialize_ray, query_frontend): model_name = "identity" query_frontend.add_model.remote(model_name, IdentityModelActor) result_oids = [] inputs = list(range(10)) for i in inputs: result_oids.append(send_predict(query_frontend, model_name, i)) result = ray.get(result_oids) assert result == inputs
def test_replicas(initialize_ray, query_frontend): model_name = "sleep_replicated" query_frontend.add_model.remote(model_name, SleepModelActor) query_frontend.scale_up_model.remote(model_name, SleepModelActor, 3) result_oids = [] inputs = list(range(30)) for i in inputs: result_oids.append(send_predict(query_frontend, model_name, i)) ray.get(result_oids) metrics = ray.get(query_frontend.get_metric.remote()) for (name, replica_id), metric in metrics.items(): if name == model_name: assert len(metric["batch_size"]) != 0
def test_batching(initialize_ray, query_frontend): # We use a sleep actor to test batching is the model queue has # time to build up model_name = "sleep" query_frontend.add_model.remote(model_name, SleepModelActor) result_oids = [] inputs = list(range(10)) for i in inputs: result_oids.append(send_predict(query_frontend, model_name, i)) ray.get(result_oids) metrics = ray.get(query_frontend.get_metric.remote())[("sleep", 0)]["batch_size"] assert 2 in metrics
def test_sklearn(initialize_ray, query_frontend): model_name = "linear_regression" query_frontend.add_model.remote(model_name, SKLearnModelActor) model = LinearRegression() X = np.ones(shape=(100, 2)) y = np.ones(100) model.fit(X, y) result_oids = [] inputs = np.ones(shape=(100, 2)) * 2 for i in inputs: result_oids.append(send_predict(query_frontend, model_name, i)) result = np.array(ray.get(result_oids)).flatten() truth = model.predict(inputs).flatten() assert np.array_equal(result, truth)
def process_request(): oids = [] for _ in range(num_replicas): oids.append(send_predict(query_frontend, model_name, "")) ray.get(oids)
def send_noop_and_wait(): ray.get(send_predict(query_frontend, model_name, ""))