def trace_model_from_checkpoint(logdir,
logger,
method_name='forward',
file='best'):
config_path = f'{logdir}/configs/_config.json'
checkpoint_path = f'{logdir}/checkpoints/{file}.pth'
logger.info('Load config')
config = safitty.load(config_path)
if 'distributed_params' in config:
del config['distributed_params']
# Get expdir name
# noinspection SpellCheckingInspection,PyTypeChecker
# We will use copy of expdir from logs for reproducibility
expdir_name = config['args']['expdir']
logger.info(f'expdir_name from args: {expdir_name}')
sys.path.insert(0, os.path.abspath(join(logdir, '../')))
expdir_from_logs = os.path.abspath(join(logdir, '../', expdir_name))
logger.info(f'expdir_from_logs: {expdir_from_logs}')
logger.info('Import experiment and runner from logdir')
ExperimentType, RunnerType = \
import_experiment_and_runner(Path(expdir_from_logs))
experiment: Experiment = ExperimentType(config)
logger.info(f'Load model state from checkpoints/{file}.pth')
model = experiment.get_model(next(iter(experiment.stages)))
checkpoint = utils.load_checkpoint(checkpoint_path)
utils.unpack_checkpoint(checkpoint, model=model)
device = 'cpu'
stage = list(experiment.stages)[0]
loader = 0
mode = 'eval'
requires_grad = False
opt_level = None
runner: RunnerType = RunnerType()
runner.model, runner.device = model, device
batch = experiment.get_native_batch(stage, loader)
logger.info('Tracing')
traced = trace_model(
model,
runner,
batch,
method_name=method_name,
mode=mode,
requires_grad=requires_grad,
opt_level=opt_level,
device=device,
)
logger.info('Done')
return traced
def trace_model_from_checkpoint(
logdir: Path,
method_name: str,
checkpoint_name: str,
stage: str = None,
loader: Union[str, int] = None,
mode: str = "eval",
requires_grad: bool = False,
opt_level: str = None,
device: Device = "cpu",
):
"""
Traces model using created experiment and runner.
Args:
logdir (Union[str, Path]): Path to Catalyst logdir with model
checkpoint_name (str): Name of model checkpoint to use
stage (str): experiment's stage name
loader (Union[str, int]): experiment's loader name or its index
method_name (str): Model's method name that will be
used as entrypoint during tracing
mode (str): Mode for model to trace (``train`` or ``eval``)
requires_grad (bool): Flag to use grads
opt_level (str): AMP FP16 init level
device (str): Torch device
Returns:
the traced model
"""
config_path = logdir / "configs" / "_config.json"
checkpoint_path = logdir / "checkpoints" / f"{checkpoint_name}.pth"
print("Load config")
config: Dict[str, dict] = load_config(config_path)
runner_params = config.get("runner_params", {}) or {}
# Get expdir name
config_expdir = Path(config["args"]["expdir"])
# We will use copy of expdir from logs for reproducibility
expdir = Path(logdir) / "code" / config_expdir.name
print("Import experiment and runner from logdir")
ExperimentType, RunnerType = import_experiment_and_runner(expdir)
experiment: ConfigExperiment = ExperimentType(config)
print(f"Load model state from checkpoints/{checkpoint_name}.pth")
if stage is None:
stage = list(experiment.stages)[0]
model = experiment.get_model(stage)
checkpoint = load_checkpoint(checkpoint_path)
unpack_checkpoint(checkpoint, model=model)
runner: RunnerType = RunnerType(**runner_params)
runner.model, runner.device = model, device
if loader is None:
loader = 0
batch = get_native_batch_from_loaders(
loaders=experiment.get_loaders(stage), loader=loader)
# function to run prediction on batch
def predict_fn(model, inputs, **kwargs):
_model = runner.model
runner.model = model
result = runner.predict_batch(inputs, **kwargs)
runner.model = _model
return result
print("Tracing")
traced_model = trace_model(
model=model,
predict_fn=predict_fn,
batch=batch,
method_name=method_name,
mode=mode,
requires_grad=requires_grad,
opt_level=opt_level,
device=device,
)
print("Done")
return traced_model
def work(self):
args, config = self.parse_args_uargs()
set_global_seed(args.seed)
Experiment, R = import_experiment_and_runner(Path(args.expdir))
runner_params = config.pop('runner_params', {})
experiment = Experiment(config)
runner: Runner = R(**runner_params)
self.experiment = experiment
self.runner = runner
stages = experiment.stages[:]
if self.task.parent:
self.parent = self.task_provider.by_id(self.task.parent)
if self.master:
task = self.get_parent_task()
task.steps = len(stages)
self.task_provider.commit()
self._checkpoint_fix_config(experiment)
_get_callbacks = experiment.get_callbacks
def get_callbacks(stage):
res = self.callbacks()
for k, v in _get_callbacks(stage).items():
res[k] = v
self._checkpoint_fix_callback(res)
return res
experiment.get_callbacks = get_callbacks
if experiment.logdir is not None:
dump_environment(config, experiment.logdir, args.configs)
if self.distr_info:
info = yaml_load(self.task.additional_info)
info['resume'] = {
'master_computer': self.distr_info['master_computer'],
'master_task_id': self.task.id - self.distr_info['rank'],
'load_best': True
}
self.task.additional_info = yaml_dump(info)
self.task_provider.commit()
experiment.stages_config = {
k: v
for k, v in experiment.stages_config.items()
if k == experiment.stages[0]
}
runner.run_experiment(experiment, check=args.check)
if runner.state.exception:
raise runner.state.exception
if self.master and self.trace:
traced = trace_model_from_checkpoint(self.experiment.logdir, self)
torch.jit.save(traced, self.trace)
return {'stage': experiment.stages[-1], 'stages': stages}