def __init__(self, vertex_config, model_name):
super().__init__()
handle_list = list()
for node_id in vertex_config.keys():
backend_config = vertex_config[node_id]
with serve_reference.using_router(node_id):
serve_reference.create_endpoint(node_id)
config = serve_reference.BackendConfig(**backend_config)
if node_id == "prepoc":
min_img_size = 224
transform = transforms.Compose([
transforms.Resize(min_img_size),
transforms.ToTensor(),
transforms.Normalize(
mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225],
),
])
serve_reference.create_backend(Transform,
node_id,
transform,
backend_config=config)
elif node_id == "model":
serve_reference.create_backend(
PredictModelPytorch,
node_id,
model_name,
True,
backend_config=config,
)
serve_reference.link(node_id, node_id)
handle_list.append(serve_reference.get_handle(node_id))
self.chain_handle = ChainHandle(handle_list)
def main(batch_size, num_warmups, num_queries, return_type):
serve_reference.init()
def noop(_, *args, **kwargs):
bs = serve_reference.context.batch_size
assert (bs == batch_size
), f"worker received {bs} which is not what expected"
result = ""
if return_type == "torch":
result = torch.zeros((3, 224, 224))
if bs is None: # No batching
return result
else:
return [result] * bs
if batch_size:
noop = serve_reference.accept_batch(noop)
with serve_reference.using_router("noop"):
serve_reference.create_endpoint("noop", "/noop")
config = serve_reference.BackendConfig(max_batch_size=batch_size)
serve_reference.create_backend(noop, "noop", backend_config=config)
serve_reference.link("noop", "noop")
handle = serve_reference.get_handle("noop")
latency = []
for i in tqdm(range(num_warmups + num_queries)):
if i == num_warmups:
serve_reference.clear_trace()
start = time.perf_counter()
if not batch_size:
ray.get(
# This is how to pass a higher level metadata to the tracing
# context
handle.options(tracing_metadata={
"demo": "pipeline-id"
}).remote())
else:
ray.get(handle.enqueue_batch(val=[1] * batch_size))
# ray.get([handle.remote() for _ in range(batch_size)])
end = time.perf_counter()
latency.append(end - start)
# Remove initial samples
latency = latency[num_warmups:]
series = pd.Series(latency) * 1000
print("Latency for single noop backend (ms)")
print(series.describe(percentiles=[0.5, 0.9, 0.95, 0.99]))
_, trace_file = tempfile.mkstemp(suffix=".json")
with open(trace_file, "w") as f:
json.dump(serve_reference.get_trace(), f)
print(f"trace file written to {trace_file}")
def __init__(self, max_batch_size, pipeline_length):
self.plength = pipeline_length
self.handles = list()
for index in range(self.plength):
node_id = f"service-{index}"
with serve_reference.using_router(node_id):
serve_reference.create_endpoint(node_id)
config = serve_reference.BackendConfig(
max_batch_size=max_batch_size, num_replicas=1)
serve_reference.create_backend(noop,
node_id,
backend_config=config)
serve_reference.link(node_id, node_id)
self.handles.append(serve_reference.get_handle(node_id))
def main():
TAG = "Resnet18"
min_img_size = 224
transform = transforms.Compose([
transforms.Resize(min_img_size),
transforms.ToTensor(),
transforms.Normalize(
mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225],
),
])
for num_replica in range(1, 9):
# initialize serve
serve_reference.init(start_server=False)
serve_handle = None
with serve_reference.using_router(TAG):
serve_reference.create_endpoint(TAG)
config = serve_reference.BackendConfig(max_batch_size=8,
num_replicas=num_replica,
num_gpus=1)
serve_reference.create_backend(
PredictModelPytorch,
TAG,
transform,
"resnet18",
True,
backend_config=config,
)
serve_reference.link(TAG, TAG)
serve_handle = serve_reference.get_handle(TAG)
img = base64.b64encode(open("elephant.jpg", "rb").read())
# warmup
ready_refs, _ = ray.wait(
[serve_handle.remote(data=img) for _ in range(200)], 200)
complete_oids, _ = ray.wait(ray.get(ready_refs), num_returns=200)
del ready_refs
del complete_oids
qps = throughput_calculation(serve_handle, {"data": img}, 2000)
print(f"[Resnet18] Batch Size: 8 Replica: {num_replica} "
f"Throughput: {qps} QPS")
serve_reference.shutdown()
def get_backend_config(self, backend_tag=None):
with serve_reference.using_router(self.endpoint_name):
backend_tag = self._ensure_backend_unique(backend_tag)
return serve_reference.get_backend_config(backend_tag)
def set_max_batch_size(self, new_max_batch_size, backend_tag=None):
with serve_reference.using_router(self.endpoint_name):
backend_tag = self._ensure_backend_unique(backend_tag)
config = serve_reference.get_backend_config(backend_tag)
config.max_batch_size = new_max_batch_size
serve_reference.set_backend_config(backend_tag, config)
def scale(self, new_num_replicas, backend_tag=None):
with serve_reference.using_router(self.endpoint_name):
backend_tag = self._ensure_backend_unique(backend_tag)
config = serve_reference.get_backend_config(backend_tag)
config.num_replicas = new_num_replicas
serve_reference.set_backend_config(backend_tag, config)
def main(num_replicas, method):
for node_id in ["up", "down"]:
with serve_reference.using_router(node_id):
serve_reference.create_endpoint(node_id)
config = serve_reference.BackendConfig(max_batch_size=1,
num_replicas=num_replicas)
serve_reference.create_backend(noop,
node_id,
backend_config=config)
serve_reference.link(node_id, node_id)
with serve_reference.using_router("up"):
up_handle = serve_reference.get_handle("up")
with serve_reference.using_router("down"):
down_handle = serve_reference.get_handle("down")
start = time.perf_counter()
oids = []
if method == "chain":
for i in range(num_queries):
r = up_handle.options(tracing_metadata={
"pipeline-id": i
}).remote(sleep_time=0.01, data=image_data)
r = down_handle.options(tracing_metadata={
"pipeline-id": i
}).remote(
sleep_time=0.02,
data=r # torch tensor
)
oids.append(r)
elif method == "group":
res = [
up_handle.options(tracing_metadata={
"pipeline-id": i
}).remote(sleep_time=0.01, data=image_data)
for i in range(num_queries)
]
oids = [
down_handle.options(tracing_metadata={
"pipeline-id": i
}).remote(
sleep_time=0.02,
data=d # torch tensor
) for i, d in enumerate(res)
]
else:
raise RuntimeError("Unreachable")
print(f"Submission time {time.perf_counter() - start}")
ray.wait(oids, len(oids))
end = time.perf_counter()
duration = end - start
qps = num_queries / duration
print(f"Throughput {qps}")
_, trace_file = tempfile.mkstemp(suffix=".json")
with open(trace_file, "w") as f:
json.dump(serve_reference.get_trace(), f)
print(f"trace file written to {trace_file}")