def test_evaluation_is_shown(): model, nested_loaders, datasets, project = synthenic_dataset_preparation() runner = project.runner(model=model, runner_name='default_experiment') evaluation = runner.evaluate() accuracy_df = evaluation[evaluation['metric_name'] == 'accuracy'] assert np.alltrue(accuracy_df['metric_res'] > 0.98) pd.set_option('display.max_columns', None)
def test_simple_visitor(): model, nested_loaders, datasets, project = synthenic_dataset_preparation() runner = project.runner(model=model, runner_name='default_experiment') flow = project.get_full_flow() class TuningVisitor(LeafVisitor): def __init__(self, task, prefix): super().__init__(task, prefix) def full_result(self, preds, targets): return TunedParams(self.decoder.tune(preds, targets)) def empty_result(self): return TunedParams({self.path: {}}) def preconditioned_result(self, preds, targets): tuned_params = self.decoder.tune(preds, targets) return TunedParams({self.path: tuned_params}) @dataclass class TunedParams(VisitorOut): data: Dict = field(default_factory=lambda: {}) def __iadd__(self, other): self.data.update(other.data) return self def reduce(self): return self.data visitor = RootCompositeVisitor(flow, TuningVisitor, TunedParams) predictions, targets, intepretations = runner.load_inference_results() res = visitor(predictions['valid'], targets['valid']) print(res)
def test_load_tuned_pipeline_from_decoder(): model, nested_loaders, datasets, project = synthenic_dataset_preparation() runner = project.runner(model=model, runner_name='default_experiment') tuned_params = torch.load(runner.project_dir / runner.default_logdir / 'tuned_params.pkl') flow = project.get_full_flow() flow.get_decoder().load_tuned(tuned_params)
def test_composite_filtering(): model, nested_loaders, datasets, project = synthenic_dataset_preparation() runner = project.runner(model=model, runner_name='default_experiment') flow = project.get_full_flow() filter_func = flow.get_filter() predictions, targets, interpretations = runner.load_inference_results() filtered_results = filter_func(predictions['test'], targets['test']) print(filtered_results)
def test_interpretation_default_runner(): model, nested_loaders, datasets, project = synthenic_dataset_preparation() runner = project.runner(model=model, runner_name='default_experiment') model = runner.best() predictions, targets, interpretations = runner.infer(model=model) print(interpretations) print(predictions)
def test_synthetic_dataset_missing_values(): model, nested_loaders, datasets, project = synthenic_dataset_preparation() dataset = datasets['train'] for i in range(32): X, y = dataset[i] assert 'gt' in X assert '_availability' in X['gt'] if y['camera_blocked'].item() < 0: assert not X['gt']['_availability']['camera_blocked'].item()
def test_synthetic_dataset_default_runner(): model, nested_loaders, datasets, project = synthenic_dataset_preparation() runner = project.runner(model=model, runner_name='default_experiment') flow: TaskFlow = project.get_full_flow() criterion = flow.get_loss() assert len(criterion.get_metrics()) < 100, 'Callbacks are too many!' runner.train(num_epochs=10) early_stop_callback = runner.default_callbacks[-1] assert early_stop_callback.best_score >= 0, 'Negative loss function!' print_any_prediction(criterion, model, nested_loaders, runner)
def test_synthetic_dataset(): model, nested_loaders, datasets, project = synthenic_dataset_preparation() runner = project.runner(model=model, runner_name='security_logs') flow: TaskFlow = project.get_full_flow() criterion = flow.get_loss() runner.train( model=model, criterion=criterion, optimizer=optim.AdamW(model.parameters(), lr=1e-4), loaders=nested_loaders, callbacks=[], num_epochs=2, ) print_any_prediction(criterion, model, nested_loaders, runner)
def test_inference_synthetic(): model, nested_loaders, datasets, project = synthenic_dataset_preparation() runner = project.runner(model=model, runner_name='default_experiment') model = runner.best() n = 4 * torch.cuda.device_count() flow: TaskFlow = project.get_full_flow() dataset = flow.get_dataset() loader = DataLoader(dataset, batch_size=n, shuffle=False) X, y = next(iter(loader)) del X['gt'] model = model.eval() res = model(X) treelib_explainer = flow.get_treelib_explainer() tree = treelib_explainer(res) tree.show()
def test_interpretation_synthetic(): model, nested_loaders, datasets, project = synthenic_dataset_preparation() runner = project.runner(model=model, runner_name='default_experiment') flow = project.get_full_flow() loaders = OrderedDict({'infer': nested_loaders['valid']}) model = runner.best() tensorboard_converters = TensorboardConverters( logdir=runner.project_dir / runner.default_logdir, tensorboard_loggers=TensorboardConverter(), datasets=datasets) callbacks = OrderedDict([("interpretation", InterpretationCallback(flow, tensorboard_converters)), ("inference", InferDictCallback())]) r = runner.infer(loaders=loaders, callbacks=callbacks) interpretations = callbacks["interpretation"].interpretations print(interpretations)
def test_repr_flow(): model, nested_loaders, datasets, project = synthenic_dataset_preparation() flow = project.get_full_flow() print(flow)
def test_load_tuned_pipeline(): model, nested_loaders, datasets, project = synthenic_dataset_preparation() runner = project.runner(model=model, runner_name='default_experiment') tuned_params = runner.load_tuned() print(tuned_params)