def test_step_does_not_change_input_model(self, method):
dataset = 'CUB'
backbone = 'Conv4'
args = dict(dataset=dataset,
backbone=backbone,
method=method,
train_aug=True,
n_iter=2)
path_to_model = os.path.join(
self.current_dir,
path_to_step_output(dataset, backbone, method, 'tests_data'),
'best_model.tar')
model_1 = FetchModel(path_to_model).apply()
path_to_features = os.path.join(
self.current_dir,
path_to_step_output(dataset, backbone, method, 'tests_data'),
'novel.hdf5')
features, labels = load_features_and_labels_from_file(path_to_features)
MethodEvaluation(**args).apply(model_1, (features, labels))
model_2 = FetchModel(path_to_model).apply()
assert model_1['epoch'] == model_2['epoch']
assert model_1['state'].keys() == model_2['state'].keys()
for key in model_1['state'].keys():
assert model_1['state'][key].equal(model_2['state'][key])
def test_step_returns_epoch_and_state_dictionnary(self):
input_path = os.path.join(self.current_dir, path_to_step_output('CUB', 'Conv4', 'baseline', 'tests_data'),
'0.tar')
output = FetchModel(input_path).apply()
assert output.keys() == {'epoch', 'state'}
def test_step_does_not_return_error(self, method):
dataset = 'CUB'
backbone = 'Conv4'
args = dict(
dataset=dataset,
backbone=backbone,
method=method,
train_aug=True,
n_iter=2,
n_swaps=2,
)
path_to_model = os.path.join(
self.current_dir,
path_to_step_output(dataset, backbone, method, 'tests_data'),
'0.tar')
model = FetchModel(path_to_model).apply()
path_to_features = os.path.join(
self.current_dir,
path_to_step_output(dataset, backbone, method, 'tests_data'),
'novel.hdf5')
features, labels = load_features_and_labels_from_file(path_to_features)
results = MethodEvaluation(**args).apply(model, (features, labels))
def test_returns_same_acc_mean_on_same_model_when_random_seed_is_the_same(
self, method):
dataset = 'CUB'
backbone = 'Conv4'
args = dict(
dataset=dataset,
backbone=backbone,
method=method,
train_aug=True,
n_iter=2,
random_seed=1,
)
path_to_model = os.path.join(
self.current_dir,
path_to_step_output(dataset, backbone, method, 'tests_data'),
'best_model.tar')
model_state = FetchModel(path_to_model).apply()
path_to_features = os.path.join(
self.current_dir,
path_to_step_output(dataset, backbone, method, 'tests_data'),
'novel.hdf5')
features, labels = load_features_and_labels_from_file(path_to_features)
results_1 = MethodEvaluation(**args).apply(model_state,
(features, labels))
results_2 = MethodEvaluation(**args).apply(model_state,
(features, labels))
assert results_1 == results_2
def test_output_is_the_same_when_input_is_the_same(method):
current_dir = os.path.dirname(__file__)
dataset = 'CUB'
backbone = 'Conv4'
path = os.path.join(
current_dir,
path_to_step_output(dataset, backbone, method, 'tests_data'),
'0.tar')
args = dict(
dataset=dataset,
backbone=backbone,
method=method,
train_aug=True,
shallow=False,
output_dir=os.path.join(current_dir, 'tests_data'),
random_seed=1,
)
model = FetchModel(path).apply()
features1, labels1 = Embedding(**args).apply(model)
features2, labels2 = Embedding(**args).apply(model)
assert np.array_equal(features1, features2)
def test_step_does_not_return_error_for_maml(self, method):
dataset = 'CUB'
backbone = 'Conv4'
args = dict(dataset=dataset,
backbone=backbone,
method=method,
train_aug=True,
n_iter=2)
path_to_model = os.path.join(
self.current_dir,
path_to_step_output(dataset, backbone, method, 'tests_data'),
'best_model.tar')
model = FetchModel(path_to_model).apply()
results = MethodEvaluation(**args).apply(model, None)
def test_step_does_not_return_error(method):
current_dir = os.path.dirname(__file__)
dataset = 'CUB'
backbone = 'Conv4'
path = os.path.join(
current_dir,
path_to_step_output(dataset, backbone, method, 'tests_data'),
'0.tar')
args = dict(
dataset=dataset,
backbone=backbone,
method=method,
train_aug=True,
shallow=True,
output_dir=os.path.join(current_dir, 'tests_data'),
)
model = FetchModel(path).apply()
features, labels = Embedding(**args).apply(model)
def test_step_admits_plus_character(self):
input_path = os.path.join(self.current_dir, path_to_step_output('CUB', 'Conv4', 'baseline++', 'tests_data'),
'0.tar')
output = FetchModel(input_path).apply()
assert output.keys() == {'epoch', 'state'}