def test_evaluation_is_shown():
model, nested_loaders, datasets, project = synthenic_dataset_preparation()
runner = project.runner(model=model, runner_name='default_experiment')
evaluation = runner.evaluate()
accuracy_df = evaluation[evaluation['metric_name'] == 'accuracy']
assert np.alltrue(accuracy_df['metric_res'] > 0.98)
pd.set_option('display.max_columns', None)
def test_simple_visitor():
model, nested_loaders, datasets, project = synthenic_dataset_preparation()
runner = project.runner(model=model, runner_name='default_experiment')
flow = project.get_full_flow()
class TuningVisitor(LeafVisitor):
def __init__(self, task, prefix):
super().__init__(task, prefix)
def full_result(self, preds, targets):
return TunedParams(self.decoder.tune(preds, targets))
def empty_result(self):
return TunedParams({self.path: {}})
def preconditioned_result(self, preds, targets):
tuned_params = self.decoder.tune(preds, targets)
return TunedParams({self.path: tuned_params})
@dataclass
class TunedParams(VisitorOut):
data: Dict = field(default_factory=lambda: {})
def __iadd__(self, other):
self.data.update(other.data)
return self
def reduce(self):
return self.data
visitor = RootCompositeVisitor(flow, TuningVisitor, TunedParams)
predictions, targets, intepretations = runner.load_inference_results()
res = visitor(predictions['valid'], targets['valid'])
print(res)
def test_load_tuned_pipeline_from_decoder():
model, nested_loaders, datasets, project = synthenic_dataset_preparation()
runner = project.runner(model=model, runner_name='default_experiment')
tuned_params = torch.load(runner.project_dir / runner.default_logdir /
'tuned_params.pkl')
flow = project.get_full_flow()
flow.get_decoder().load_tuned(tuned_params)
def test_composite_filtering():
model, nested_loaders, datasets, project = synthenic_dataset_preparation()
runner = project.runner(model=model, runner_name='default_experiment')
flow = project.get_full_flow()
filter_func = flow.get_filter()
predictions, targets, interpretations = runner.load_inference_results()
filtered_results = filter_func(predictions['test'], targets['test'])
print(filtered_results)
def test_interpretation_default_runner():
model, nested_loaders, datasets, project = synthenic_dataset_preparation()
runner = project.runner(model=model, runner_name='default_experiment')
model = runner.best()
predictions, targets, interpretations = runner.infer(model=model)
print(interpretations)
print(predictions)
def test_synthetic_dataset_missing_values():
model, nested_loaders, datasets, project = synthenic_dataset_preparation()
dataset = datasets['train']
for i in range(32):
X, y = dataset[i]
assert 'gt' in X
assert '_availability' in X['gt']
if y['camera_blocked'].item() < 0:
assert not X['gt']['_availability']['camera_blocked'].item()
def test_synthetic_dataset_default_runner():
model, nested_loaders, datasets, project = synthenic_dataset_preparation()
runner = project.runner(model=model, runner_name='default_experiment')
flow: TaskFlow = project.get_full_flow()
criterion = flow.get_loss()
assert len(criterion.get_metrics()) < 100, 'Callbacks are too many!'
runner.train(num_epochs=10)
early_stop_callback = runner.default_callbacks[-1]
assert early_stop_callback.best_score >= 0, 'Negative loss function!'
print_any_prediction(criterion, model, nested_loaders, runner)
def test_synthetic_dataset():
model, nested_loaders, datasets, project = synthenic_dataset_preparation()
runner = project.runner(model=model, runner_name='security_logs')
flow: TaskFlow = project.get_full_flow()
criterion = flow.get_loss()
runner.train(
model=model,
criterion=criterion,
optimizer=optim.AdamW(model.parameters(), lr=1e-4),
loaders=nested_loaders,
callbacks=[],
num_epochs=2,
)
print_any_prediction(criterion, model, nested_loaders, runner)
def test_inference_synthetic():
model, nested_loaders, datasets, project = synthenic_dataset_preparation()
runner = project.runner(model=model, runner_name='default_experiment')
model = runner.best()
n = 4 * torch.cuda.device_count()
flow: TaskFlow = project.get_full_flow()
dataset = flow.get_dataset()
loader = DataLoader(dataset, batch_size=n, shuffle=False)
X, y = next(iter(loader))
del X['gt']
model = model.eval()
res = model(X)
treelib_explainer = flow.get_treelib_explainer()
tree = treelib_explainer(res)
tree.show()
def test_interpretation_synthetic():
model, nested_loaders, datasets, project = synthenic_dataset_preparation()
runner = project.runner(model=model, runner_name='default_experiment')
flow = project.get_full_flow()
loaders = OrderedDict({'infer': nested_loaders['valid']})
model = runner.best()
tensorboard_converters = TensorboardConverters(
logdir=runner.project_dir / runner.default_logdir,
tensorboard_loggers=TensorboardConverter(),
datasets=datasets)
callbacks = OrderedDict([("interpretation",
InterpretationCallback(flow,
tensorboard_converters)),
("inference", InferDictCallback())])
r = runner.infer(loaders=loaders, callbacks=callbacks)
interpretations = callbacks["interpretation"].interpretations
print(interpretations)
def test_repr_flow():
model, nested_loaders, datasets, project = synthenic_dataset_preparation()
flow = project.get_full_flow()
print(flow)
def test_load_tuned_pipeline():
model, nested_loaders, datasets, project = synthenic_dataset_preparation()
runner = project.runner(model=model, runner_name='default_experiment')
tuned_params = runner.load_tuned()
print(tuned_params)