def _run_local(self):
_LOGGER.info(
"adding data file(s) to project_id {} from file path {}".format(
self.project_id, self.uri
)
)
if os.path.isdir(self.uri):
_LOGGER.info("Path {} is directory".format(self.uri))
yield JobProgressSchema().dump(
{"iter_indefinite": False, "iter_val": 0.0, "iter_class": "copy_folder"}
)
for progress, _ in self._run_copy_folder(self.uri):
yield JobProgressSchema().dump(
{
"iter_indefinite": False,
"iter_val": progress,
"iter_class": "copy_folder",
}
)
else:
_LOGGER.info("Path {} is file".format(self.uri))
yield JobProgressSchema().dump(
{"iter_indefinite": True, "iter_class": "copy"}
)
self._run_copy_file(self.uri, self._get_project_data())
def _run_pruning_sensitivity(
self,
model: ProjectModel,
data: DataLoader,
profile: ProjectLossProfile,
num_steps: int,
):
analysis = None
for (analysis, progress) in pruning_perf_sens_one_shot_iter(
model.file_path,
data,
self.batch_size,
self.core_count,
self.iterations_per_check,
self.warmup_iterations_per_check,
optimization_level=
0, # TODO: fix timings so optimization_level=1 works
iters_sleep_time=
0.1, # hack for current issue with deepsparse and GIL
):
yield JobProgressSchema().dump({
"iter_indefinite": False,
"iter_class": "analysis",
"iter_val": progress.val,
"num_steps": num_steps,
"step_index": 1, # baseline always runs before
"step_class": "pruning_estimation",
})
# update but do not save until everything is completed so we don't hammer the DB
CreatePerfProfileJobWorker.add_profile_pruning_results(
profile, analysis.results_model, analysis.results)
def _run_baseline_perf(
self,
model: ProjectModel,
data: DataLoader,
profile: ProjectLossProfile,
num_steps: int,
):
analysis = None
for (analysis, progress) in pruning_perf_sens_one_shot_iter(
model.file_path,
data,
self.batch_size,
self.core_count,
self.iterations_per_check,
self.warmup_iterations_per_check,
sparsity_levels=[0.0],
optimization_level=1,
):
yield JobProgressSchema().dump({
"iter_indefinite": False,
"iter_class": "analysis",
"iter_val": progress.val,
"num_steps": num_steps,
"step_index": 0, # baseline always runs first
"step_class": "baseline_estimation",
})
# update but do not save until everything is completed so we don't hammer the DB
CreatePerfProfileJobWorker.add_profile_baseline_results(
profile, analysis.results_model, analysis.results)
def _run_download(self) -> Iterator[Dict[str, Any]]:
_LOGGER.info(
("adding model file to project_id {} and "
"model_id {} from url {}").format(self.project_id, self.model_id,
self.uri))
# yield start progress to mark the expected flow
yield JobProgressSchema().dump({
"iter_indefinite": False,
"iter_class": "download",
"iter_val": 0.0
})
model = self._get_project_model()
with NamedTemporaryFile() as temp:
temp_path = os.path.join(gettempdir(), temp.name)
for download_progress in download_file_iter(self._uri,
temp_path,
overwrite=True):
progress_val = (float(download_progress.downloaded) /
float(download_progress.content_length)
if download_progress.content_length else None)
yield JobProgressSchema().dump({
"iter_indefinite": False,
"iter_class": "download",
"iter_val": progress_val,
})
ModelFromPathJobWorker._save_project_model(model, temp_path)
_LOGGER.info("added model file to project_id {} and "
"model_id {} from url {}".format(self.project_id,
self.model_id, self.uri))
def _run_baseline_loss(self, model: ProjectModel,
profile: ProjectLossProfile, num_steps: int):
analysis = None
for (analysis, progress) in pruning_loss_sens_magnitude_iter(
model.file_path, [0.0]):
yield JobProgressSchema().dump({
"iter_indefinite": False,
"iter_class": "analysis",
"iter_val": progress.val,
"num_steps": num_steps,
"step_index": 0, # baseline always runs first
"step_class": "baseline_estimation",
})
# update but do not save until everything is completed so we don't hammer the DB
CreateLossProfileJobWorker.add_profile_baseline_results(
profile, analysis.results_model, analysis.results)
def _run_weight_magnitude_pruning_sensitivity(self, model: ProjectModel,
profile: ProjectLossProfile,
num_steps: int):
if self.pruning_structure != "unstructured":
raise ValueError(
"pruning_structure of {} is not currently supported".format(
self.pruning_structure))
analysis = None
for (analysis,
progress) in pruning_loss_sens_magnitude_iter(model.file_path):
yield JobProgressSchema().dump({
"iter_indefinite": False,
"iter_class": "analysis",
"iter_val": progress.val,
"num_steps": num_steps,
"step_index": 1, # baseline always runs before
"step_class": "pruning_estimation",
})
# update but do not save until everything is completed so we don't hammer the DB
CreateLossProfileJobWorker.add_profile_pruning_results(
profile, analysis.results_model, analysis.results)
def _run_local(self) -> Iterator[Dict[str, Any]]:
_LOGGER.info(
("adding model file to project_id {} and "
"model_id {} from file path {}").format(self.project_id,
self.model_id, self.uri))
# yield start progress to mark the expected flow
yield JobProgressSchema().dump({
"iter_indefinite": True,
"iter_class": "copy"
})
model = self._get_project_model()
if not os.path.exists(self._uri):
raise ValueError("local path of {} does not exist".format(
self._uri))
ModelFromPathJobWorker._save_project_model(model, self._uri)
_LOGGER.info(
("added model file to project_id {} and "
"model_id {} from file path {}").format(self.project_id,
self.model_id, self.uri))
def run(self) -> Iterator[Dict[str, Any]]:
"""
Perform the work for the job.
Runs and saves the appropriate benchmark based on the configuration
:return: an iterator containing progress update information
"""
_LOGGER.info(("running benchmark for project_id {} and "
"model_id {} and benchmark_id {} with "
"core_counts:{}, batch sizes:{} "
"instruction_sets:{}, inference_models:{} ").format(
self.project_id,
self.model_id,
self.benchmark_id,
self.core_counts,
self.batch_sizes,
self.instruction_sets,
self.inference_models,
))
project_model = self._get_project_model()
project_model.validate_filesystem()
benchmark = self._get_project_benchmark()
benchmark.result = {"benchmarks": []}
sys_info = get_ml_sys_info()
cores_per_socket = (sys_info["cores_per_socket"]
if "cores_per_socket" in sys_info else 1)
num_sockets = sys_info[
"num_sockets"] if "num_sockets" in sys_info else 1
max_cores = cores_per_socket * num_sockets
optims = set()
for inference_model in self.inference_models:
inference_model_optimization = inference_model[
"inference_model_optimization"]
if inference_model_optimization:
optims.add(inference_model_optimization)
iterables = [
x for x in itertools.product(self.core_counts, self.batch_sizes,
self.inference_models)
]
num_steps = len(iterables) + len(optims)
step_index = 0
pruned_models = {}
for inference_model_optimization in optims:
model_proto = check_load_model(project_model.file_path)
for progress in self._get_pruned_model_proto(
model_proto, inference_model_optimization):
yield JobProgressSchema().dump({
"iter_indefinite":
False,
"iter_class":
"benchmark",
"iter_val": (step_index + progress) / num_steps,
"num_steps":
num_steps,
"step_index":
step_index,
"step_class":
"pruning_{}".format(inference_model_optimization),
})
pruned_models[inference_model_optimization] = model_proto
step_index += 1
for core_count, batch_size, inference_model in iterables:
inference_engine = inference_model["inference_engine"]
inference_model_optimization = inference_model[
"inference_model_optimization"]
model = project_model.file_path
if inference_model_optimization:
model = pruned_models[inference_model_optimization]
if inference_engine == ORT_CPU_ENGINE and (core_count == max_cores
or core_count < 1):
runner = ORTModelRunner(model, batch_size=batch_size)
elif inference_engine == ORT_CPU_ENGINE and (
core_count != max_cores and core_count > 0):
_LOGGER.error(
"Can only run onnxruntime with max core count of {}".
format(max_cores))
raise Exception(
"Can only run onnxruntime with max core count of {}".
format(max_cores))
elif inference_engine == DEEPSPARSE_ENGINE:
runner = DeepSparseModelRunner(model, batch_size, core_count)
elif inference_engine == ORT_GPU_ENGINE:
raise NotImplementedError()
else:
raise ValueError(
"Invalid inference engine {}".format(inference_engine))
step_class = ("{}_optim_{}_batch_size_{}_core_count_{}".format(
inference_engine,
inference_model_optimization,
batch_size,
core_count,
) if inference_model_optimization else
"{}_batch_size_{}_core_count_{}".format(
inference_engine, batch_size, core_count))
_LOGGER.debug(step_class)
for progress in self._run_benchmark(
benchmark,
model,
runner,
core_count,
batch_size,
inference_engine,
inference_model_optimization,
num_steps,
step_index,
):
yield JobProgressSchema().dump({
"iter_indefinite": False,
"iter_class": "benchmark",
"iter_val": progress,
"num_steps": num_steps,
"step_index": step_index,
"step_class": step_class,
})
step_index += 1
benchmark.save()
def _run_download(self):
_LOGGER.info(
("adding data file(s) to project_id {} from url {}").format(
self.project_id, self.uri
)
)
yield JobProgressSchema().dump(
{"iter_indefinite": False, "iter_class": "download", "iter_val": 0.0}
)
with NamedTemporaryFile() as temp:
temp_path = os.path.join(gettempdir(), temp.name)
for download_progress in download_file_iter(
self._uri, temp_path, overwrite=True
):
progress_val = (
float(download_progress.downloaded)
/ float(download_progress.content_length)
if download_progress.content_length
else None
)
yield JobProgressSchema().dump(
{
"iter_indefinite": False,
"iter_class": "download",
"iter_val": progress_val / 3 if progress_val else 0,
"step_class": "download",
"step_index": 0,
}
)
if tarfile.is_tarfile(temp_path):
_LOGGER.info("Untarring file downloaded from {}".format(self.uri))
yield JobProgressSchema().dump(
{
"iter_indefinite": False,
"iter_class": "download",
"iter_val": 1 / 3,
"step_class": "untarring",
"step_index": 1,
}
)
with TemporaryDirectory() as extract_path, tarfile.open(
temp_path, "r"
) as tar:
tar.extractall(extract_path)
yield JobProgressSchema().dump(
{
"iter_indefinite": False,
"iter_class": "download",
"iter_val": 2 / 3,
"step_class": "untarring",
"step_index": 1,
}
)
for progress, _ in self._run_copy_folder(extract_path):
yield JobProgressSchema().dump(
{
"iter_indefinite": False,
"iter_class": "download",
"iter_val": 2 / 3 + progress / 3,
"step_class": "copy_folder",
"step_index": 2,
}
)
else:
self._run_copy_file(temp_path, self._get_project_data())
yield JobProgressSchema().dump(
{
"iter_indefinite": False,
"iter_class": "download",
"iter_val": 1,
"step_class": "copy",
"step_index": 1,
}
)