def test_interpretation_synthetic():
model, nested_loaders, datasets, project = synthetic_dataset_preparation()
runner = project.runner(model=model, runner_name='default_experiment')
flow = project.get_full_flow()
loaders = OrderedDict({'infer': nested_loaders['valid']})
model = runner.best()
tensorboard_converters = TensorboardConverters(
logdir=runner.project_dir / runner.default_logdir,
tensorboard_loggers=TensorboardConverter(),
datasets=datasets)
infer_dict_callback = InferDictCallback()
callbacks = OrderedDict([
("interpretation", InterpretationCallback(flow,
tensorboard_converters)),
("inference", infer_dict_callback),
("reducer", ReplaceGatherCallback(flow, infer_dict_callback))
])
r = runner.infer(loaders=loaders, callbacks=callbacks)
interpretations = callbacks["interpretation"].interpretations
print(interpretations)
def test_load_tuned_pipeline_from_decoder():
model, nested_loaders, datasets, project = synthetic_dataset_preparation()
runner = project.runner(model=model, runner_name='default_experiment')
tuned_params = torch.load(runner.project_dir / runner.default_logdir /
'tuned_params.pkl')
flow = project.get_full_flow()
flow.get_decoder().load_tuned(tuned_params)
def test_composite_decoding():
model, nested_loaders, datasets, project = synthetic_dataset_preparation()
runner = project.runner(model=model, runner_name='default_experiment')
flow = project.get_full_flow()
decoder = flow.get_decoder()
res = runner.load_inference_results()
activated_predictions = decoder(res['logits']['test'])
print(activated_predictions)
def test_composite_filtering():
model, nested_loaders, datasets, project = synthetic_dataset_preparation()
runner = project.runner(model=model, runner_name='default_experiment')
flow = project.get_full_flow()
filter_func = flow.get_filter()
res = runner.load_inference_results()
filtered_results = filter_func(res['logits']['test'],
res['targets']['test'])
print(filtered_results)
def test_evaluation_is_shown():
model, nested_loaders, datasets, project = synthetic_dataset_preparation()
runner = project.runner(model=model, runner_name='default_experiment')
evaluation = runner.evaluate()
accuracy_df = evaluation[evaluation['metric_name'] == 'accuracy']
assert np.alltrue(accuracy_df['metric_res'] > 0.98)
mae_df = evaluation[evaluation['metric_name'] == 'mean_absolute_error']
assert np.alltrue(mae_df['metric_res'] < 5e-2)
pd.set_option('display.max_columns', None)
def test_synthetic_dataset_missing_values():
model, nested_loaders, datasets, project = synthetic_dataset_preparation()
dataset = datasets['train']
for i in range(32):
X, y = dataset[i]
assert 'gt' in X
assert '_availability' in X['gt']
if y['camera_blocked'].item() < 0:
assert not X['gt']['_availability']['camera_blocked'].item()
def test_interpretation_default_runner():
model, nested_loaders, datasets, project = synthetic_dataset_preparation()
runner = project.runner(model=model,
runner_name='default_experiment',
balance_dataparallel_memory=True)
model = runner.best()
project.converters.tensorboard_converters.type_mapping['img'] = [
img_publisher
]
project.converters.tensorboard_converters.type_mapping['text'] = [
text_publisher
]
r = runner.infer(model=model)
print(r)
def test_synthetic_dataset_default_runner():
model, nested_loaders, datasets, project = synthetic_dataset_preparation()
runner = project.runner(model=model, runner_name='default_experiment', balance_dataparallel_memory=True)
flow: TaskFlow = project.get_full_flow()
criterion = flow.get_loss()
assert len(criterion.get_metrics()) < 100, 'Callbacks are too many!'
project.converters.tensorboard_converters.type_mapping['img'] = [img_publisher]
project.converters.tensorboard_converters.type_mapping['text'] = [text_publisher]
# runner.train(num_epochs=10, callbacks=runner.default_callbacks[:1])
runner.train(num_epochs=10)
early_stop_callback = runner.default_callbacks[-1]
assert early_stop_callback.best_score >= 0, 'Negative loss function!'
print_any_prediction(criterion, model, nested_loaders, runner)
def treelib_explanation_on_first_batch():
model, nested_loaders, datasets, project = synthetic_dataset_preparation()
runner = project.runner(model=model, runner_name='default_experiment')
model = runner.best()
n = 4 * torch.cuda.device_count()
flow: TaskFlow = project.get_full_flow()
dataset = flow.get_dataset()
loader = DataLoader(dataset, batch_size=n, shuffle=False)
X, y = next(iter(loader))
del X['gt']
model = model.eval()
res = model(X)
treelib_explainer = flow.get_treelib_explainer()
tree = treelib_explainer(res)
return tree
def test_synthetic_dataset():
model, nested_loaders, datasets, project = synthetic_dataset_preparation()
runner = project.runner(model=model, runner_name='security_logs')
flow: TaskFlow = project.get_full_flow()
criterion = flow.get_loss()
args = TrainingArguments(
num_epochs=2,
callbacks=[],
loaders=nested_loaders,
optimizer=optim.Adam(model.parameters(), lr=1e-4),
)
runner.train(**args)
print_any_prediction(criterion, model, nested_loaders, runner)
def test_rf_multi_input():
model, nested_loaders, datasets, project = synthetic_dataset_preparation()
runner = project.runner(model=model,
runner_name='default_experiment',
balance_dataparallel_memory=False)
loader = nested_loaders['valid']
X, y = next(iter(loader))
X = runner.batch_to_model_device(X)
y = runner.batch_to_model_device(y)
class FeaturesModel(nn.Module):
def __init__(self, model):
super().__init__()
self.model = model
def forward(self, x):
features = self.model.seq[:4](x['syn_img'])
return {'features': features}
model = FeaturesModel(model).eval()
with torch.no_grad():
res = model(X)
out_mask = {
key: torch.zeros_like(value).bool()
for key, value in res.items()
}
out_mask['features'][:, :, 11, 11] = True
rf_mask = compute_receptive_field(model, X, out_mask)
nonzero_indices = (torch.nonzero(rf_mask['syn_img']))
mins = nonzero_indices.min(dim=0).values
maxs = nonzero_indices.max(dim=0).values + 1
lengths = maxs - mins
assert lengths[0].item() == loader.batch_size
assert lengths[1].item() == 3
assert lengths[2].item() == 9
assert lengths[3].item() == 9
def test_load_tuned_pipeline():
model, nested_loaders, datasets, project = synthetic_dataset_preparation()
runner = project.runner(model=model, runner_name='default_experiment')
tuned_params = runner.load_tuned()
print(tuned_params)
def test_repr_flow():
model, nested_loaders, datasets, project = synthetic_dataset_preparation()
flow = project.get_full_flow()
print(flow)
