def run(self) -> None:
from modelci.hub.deployer.dispatcher import serve
for job in iter(self.job_queue.get, None):
# exit the queue
if job is self._queue_finish_flag:
break
# start a new container if container not started
if job.container_name is None:
container = serve(save_path=job.model.saved_path,
device=job.device)
container_name = container.name
# remember to clean-up the created container
self._hold_container.put(container)
else:
container_name = job.container_name
# change model status
job.model.status = Status.RUNNING
ModelService.update_model(job.model)
profiler = Profiler(model_info=job.model,
server_name=container_name,
inspector=job.client)
dpr = profiler.diagnose(device=job.device)
ModelService.append_dynamic_profiling_result(job.model.id,
dynamic_result=dpr)
# set model status to pass
job.model.status = Status.PASS
ModelService.update_model(job.model)
if job.container_name is None:
# get holding container
self._hold_container.get().stop()
def test_register_dynamic_profiling_result():
model = ModelService.get_models_by_name('ResNet50')[0]
dpr = DynamicProfileResultBO(
'gpu:01', 'Tesla K40c', 1, ProfileMemory(1000, 1000, 1000),
ProfileLatency((1, 1, 1), (1, 1, 1), (1, 1, 1), (1, 1, 1)),
ProfileThroughput((1, 1, 1), (1, 1, 1), (1, 1, 1), (1, 1, 1)))
assert ModelService.append_dynamic_profiling_result(model.id, dpr)
def test_delete_dynamic_profiling_result():
model = ModelService.get_models('ResNet50')[0]
dummy_info_tuple1 = InfoTuple(avg=1, p50=1, p95=1, p99=2)
dummy_info_tuple2 = InfoTuple(avg=1, p50=1, p95=1, p99=1)
dpr = DynamicProfileResultBO(device_id='gpu:02',
device_name='Tesla K40c',
batch=1,
memory=ProfileMemory(1000, 1000, 0.5),
latency=ProfileLatency(
init_latency=dummy_info_tuple1,
preprocess_latency=dummy_info_tuple2,
inference_latency=dummy_info_tuple2,
postprocess_latency=dummy_info_tuple2,
),
throughput=ProfileThroughput(
batch_formation_throughput=1,
preprocess_throughput=1,
inference_throughput=1,
postprocess_throughput=1,
))
ModelService.append_dynamic_profiling_result(model.id, dpr)
# reload
model = ModelService.get_models('ResNet50')[0]
dpr_bo = model.profile_result.dynamic_results[0]
dpr_bo2 = model.profile_result.dynamic_results[1]
# check delete
assert ModelService.delete_dynamic_profiling_result(
model.id, dpr_bo.ip, dpr_bo.device_id)
# check result
model = ModelService.get_models('ResNet50')[0]
assert len(model.profile_result.dynamic_results) == 1
dpr_left = model.profile_result.dynamic_results[0]
assert dpr_bo2.latency.init_latency.avg == dpr_left.latency.init_latency.avg
def test_delete_dynamic_profiling_result():
model = ModelService.get_models_by_name('ResNet50')[0]
dpr = DynamicProfileResultBO(
'gpu:02', 'Tesla K40c', 1, ProfileMemory(1000, 1000, 1000),
ProfileLatency((1, 1, 2), (1, 1, 1), (1, 1, 1), (1, 1, 1)),
ProfileThroughput((1, 1, 1), (1, 1, 1), (1, 1, 1), (1, 1, 1)))
ModelService.append_dynamic_profiling_result(model.id, dpr)
# reload
model = ModelService.get_models_by_name('ResNet50')[0]
dpr_bo = model.profile_result.dynamic_results[0]
dpr_bo2 = model.profile_result.dynamic_results[1]
# check delete
assert ModelService.delete_dynamic_profiling_result(
model.id, dpr_bo.ip, dpr_bo.device_id)
# check result
model = ModelService.get_models_by_name('ResNet50')[0]
assert len(model.profile_result.dynamic_results) == 1
dpr_left = model.profile_result.dynamic_results[0]
assert dpr_bo2.latency.init_latency.avg == dpr_left.latency.init_latency.avg
def test_register_dynamic_profiling_result():
model = ModelService.get_models('ResNet50')[0]
dummy_info_tuple = InfoTuple(avg=1, p50=1, p95=1, p99=1)
dpr = DynamicProfileResultBO(device_id='gpu:01',
device_name='Tesla K40c',
batch=1,
memory=ProfileMemory(1000, 1000, 0.5),
latency=ProfileLatency(
init_latency=dummy_info_tuple,
preprocess_latency=dummy_info_tuple,
inference_latency=dummy_info_tuple,
postprocess_latency=dummy_info_tuple,
),
throughput=ProfileThroughput(
batch_formation_throughput=1,
preprocess_throughput=1,
inference_throughput=1,
postprocess_throughput=1,
))
assert ModelService.append_dynamic_profiling_result(model.id, dpr)
