def test_update_dynamic_profiling_result(): model = ModelService.get_models('ResNet50')[0] dummy_info_tuple = InfoTuple(avg=1, p50=1, p95=1, p99=1) updated_info_tuple = InfoTuple(avg=1, p50=2, p95=1, p99=1) dpr = DynamicProfileResultBO(device_id='gpu:01', device_name='Tesla K40c', batch=1, memory=ProfileMemory(1000, 2000, 0.5), latency=ProfileLatency( init_latency=dummy_info_tuple, preprocess_latency=dummy_info_tuple, inference_latency=updated_info_tuple, postprocess_latency=dummy_info_tuple, ), throughput=ProfileThroughput( batch_formation_throughput=1, preprocess_throughput=1, inference_throughput=1, postprocess_throughput=1, )) # check update assert ModelService.update_dynamic_profiling_result(model.id, dpr) # check result model = ModelService.get_models('ResNet50')[0] assert model.profile_result.dynamic_results[0].memory.memory_usage == 2000 assert model.profile_result.dynamic_results[ 0].latency.inference_latency.p50 == 2
def update_dynamic_profiling_result(cls, id_: str, dynamic_result: DynamicProfileResultBO, force_insert=False): """Update one dynamic profiling result to a model. Args: id_ (str): ID of the object dynamic_result (DynamicProfileResultBO): Dynamic profiling result force_insert: force to insert the dynamic result if it is not found Return: int: number of affected rows Raise: DoesNotExistException: Model ID does not exist in ModelDB; or dynamic profiling result to be updated does not exist and `force_insert` is not set """ id_ = ObjectId(id_) dpr_po = dynamic_result.to_dynamic_profile_result_po() pks = {'ip': dpr_po.ip, 'device_id': dpr_po.device_id} # if model ID exists if cls.__model_DAO.exists_by_id(id_): # if the dynamic profiling result to be updated exists if cls.__model_DAO.exists_dynamic_profiling_result_by_pks(id_, **pks): return cls.__model_DAO.update_dynamic_profiling_result(id_, dpr_po) # force insert is set elif force_insert: return cls.__model_DAO.register_dynamic_profiling_result(id_, dpr_po) else: raise DoesNotExistException( f'Dynamic profiling result to be updated with ip={dpr_po.ip}, ' f'device_id={dpr_po.device_id} does not exist. Either set `force_insert` or change the ip ' f'and device_id.' ) # if model ID does not exist else: raise DoesNotExistException('Model ID {} does not exist.'.format(id_))
def diagnose(self, batch_size: int = None, device='cuda', timeout=30) -> DynamicProfileResultBO: """Start diagnosing and profiling model. Args: batch_size (int): Batch size. device (str): Device name. timeout (float): Waiting for docker container timeout in second. Default timeout period is 30s. """ model_status = False retry_time = 0 # use binary exponential backoff algorithm tick = time.time() while time.time() - tick < timeout: if self.inspector.check_model_status(): model_status = True break retry_time += 1 # get backoff time in s backoff_time = random.randint(0, 2**retry_time - 1) * 1e-3 time.sleep(backoff_time) if not model_status: # raise an error as model is not served. raise ServiceException('Model not served!') if batch_size is not None: self.inspector.set_batch_size(batch_size) result = self.inspector.run_model(server_name=self.server_name, device=device) dpr_bo = DynamicProfileResultBO( ip=get_ip(), device_id=result['device_id'], device_name=result['device_name'], batch=result['batch_size'], memory=ProfileMemory( total_memory=result['total_gpu_memory'], memory_usage=result['gpu_memory_used'], utilization=result['gpu_utilization'], ), latency=ProfileLatency(inference_latency=result['latency'], ), throughput=ProfileThroughput( inference_throughput=result['total_throughput']), create_time=result['completed_time'], ) return dpr_bo
def test_register_dynamic_profiling_result(): model = ModelService.get_models('ResNet50')[0] dummy_info_tuple = InfoTuple(avg=1, p50=1, p95=1, p99=1) dpr = DynamicProfileResultBO(device_id='gpu:01', device_name='Tesla K40c', batch=1, memory=ProfileMemory(1000, 1000, 0.5), latency=ProfileLatency( init_latency=dummy_info_tuple, preprocess_latency=dummy_info_tuple, inference_latency=dummy_info_tuple, postprocess_latency=dummy_info_tuple, ), throughput=ProfileThroughput( batch_formation_throughput=1, preprocess_throughput=1, inference_throughput=1, postprocess_throughput=1, )) assert ModelService.append_dynamic_profiling_result(model.id, dpr)
def append_dynamic_profiling_result( cls, id_: str, dynamic_result: DynamicProfileResultBO): """Add one dynamic profiling result to a model. Args: id_ (str): ID of the object dynamic_result (DynamicProfileResultBO): Dynamic profiling result Return: int: number of affected rows Raises: DoesNotExistException: `model.id` does not exist in ModelDB """ id_ = ObjectId(id_) if cls.__model_DAO.exists_by_id(id_): return cls.__model_DAO.register_dynamic_profiling_result( id_, dynamic_result.to_dynamic_profile_result_po()) else: raise DoesNotExistException( 'Model ID {} does not exist.'.format(id_))
def test_delete_dynamic_profiling_result(): model = ModelService.get_models('ResNet50')[0] dummy_info_tuple1 = InfoTuple(avg=1, p50=1, p95=1, p99=2) dummy_info_tuple2 = InfoTuple(avg=1, p50=1, p95=1, p99=1) dpr = DynamicProfileResultBO(device_id='gpu:02', device_name='Tesla K40c', batch=1, memory=ProfileMemory(1000, 1000, 0.5), latency=ProfileLatency( init_latency=dummy_info_tuple1, preprocess_latency=dummy_info_tuple2, inference_latency=dummy_info_tuple2, postprocess_latency=dummy_info_tuple2, ), throughput=ProfileThroughput( batch_formation_throughput=1, preprocess_throughput=1, inference_throughput=1, postprocess_throughput=1, )) ModelService.append_dynamic_profiling_result(model.id, dpr) # reload model = ModelService.get_models('ResNet50')[0] dpr_bo = model.profile_result.dynamic_results[0] dpr_bo2 = model.profile_result.dynamic_results[1] # check delete assert ModelService.delete_dynamic_profiling_result( model.id, dpr_bo.ip, dpr_bo.device_id) # check result model = ModelService.get_models('ResNet50')[0] assert len(model.profile_result.dynamic_results) == 1 dpr_left = model.profile_result.dynamic_results[0] assert dpr_bo2.latency.init_latency.avg == dpr_left.latency.init_latency.avg