def _get_trained_network(data,
hidden_layer_sizes,
train_options=TrainOptions()):
default_train_options = TrainOptions(num_epochs=500,
optimizer="SGD",
learning_rate=0.001,
loss_function="MSELoss",
print_every=100,
use_gpu=True,
activation_function="relu",
bias=True,
dropout_rate=0.5)
option_dict = {
k: v
for k, v in train_options._asdict().items() if v is not None
}
option_dict.update({
k: None
for k, v in train_options._asdict().items() if v == "none"
})
train_options = default_train_options._replace(**option_dict)
network_options = NetworkOptions(len(data['train'][0][0]),
len(data['train'][1][0]),
hidden_layer_sizes)
network = PytorchNetwork()
network.init(data, network_options, train_options)
network.train()
return network
def run(self, networks: List[Network], data: dict, network_options: NetworkOptions, train_options: TrainOptions,
optimizer_options: OptimizerOptions):
best = {'loss': None, 'train_options': None, 'network_options': None, 'network': None}
for network in networks:
logging.info(f"Using {network.__class__.__name__}...")
# for all hidden layers
all_hidden_layer_sizes = self.hidden_layers if network_options.hidden_layer_sizes is None else \
[network_options.hidden_layer_sizes]
for hidden_layer_sizes in all_hidden_layer_sizes:
network_options = NetworkOptions(network_options.input_layer_size, network_options.output_layer_size,
hidden_layer_sizes)
# for all optimizers ...
optimizers = self.optimizers if train_options.optimizer is None else [train_options.optimizer]
for optimizer in optimizers:
optimizer = self.optimizers_map[network.__class__.__name__][optimizer]
options = train_options._replace(optimizer=optimizer)
# ... and for all loss functions ...
loss_functions = self.loss_functions if train_options.loss_function is None else \
[train_options.loss_function]
for loss_function in loss_functions:
loss_function = self.loss_functions_map[network.__class__.__name__][loss_function]
options = options._replace(loss_function=loss_function)
# ... run x times (try several times because of the random seed)
num_runs = train_options.num_runs_per_setting \
if train_options.num_runs_per_setting is not None else self.default_num_runs_per_setting
for i in range(1, num_runs + 1):
self._run_once(best, options, data, i, network, network_options, optimizer_options)
return best
def test_run_no_hidden_layers_specified_should_enumerate_over_all_hidden_layers(self):
network_mock = Mock()
network_mock.__class__.__name__ = "PytorchNetwork"
network_mock.validate.return_value = 1
optimizer = Optimizer()
optimizer.run([network_mock], {}, NetworkOptions(0, 0, None),
TrainOptions(optimizer="adam", loss_function="mse"), OptimizerOptions(None))
network_mock.init.assert_called()
self.assertEqual(optimizer.default_num_runs_per_setting * len(optimizer.hidden_layers), network_mock.init.call_count)
def test_run_should_call_network_multiple_times(self):
network_mock = Mock()
network_mock.__class__.__name__ = "PytorchNetwork"
network_mock.validate.return_value = 1
optimizer = Optimizer()
optimizer.run([network_mock], {}, NetworkOptions(0, 0, []),
TrainOptions(optimizer="adam", loss_function="mse"), OptimizerOptions(None))
network_mock.init.assert_called()
self.assertEqual(optimizer.default_num_runs_per_setting, network_mock.init.call_count)
def test_run_no_optional_options_specified_using_linear_data(self):
data = self._get_data_linear()
train_options = TrainOptions(print_every=100, use_gpu=True, activation_function="relu", num_epochs=100)
network_options = NetworkOptions(len(data['train'][0][0]), len(data['train'][1][0]), None)
network = PytorchNetwork()
optimizer = Optimizer()
best = optimizer.run([network], data, network_options, train_options, OptimizerOptions(None))
logging.info(f"Best run: {best}")
self.assertLess(best['loss'], 100)
self.assertIsNotNone(best['train_options'])
def test_load(self):
network = self._get_trained_network(self._get_data_linear(), [],
self._get_train_options_linear())
file_name = "tmp.pth"
network.save(file_name)
data = self._get_data_linear()
train_options = TrainOptions()
network_options = NetworkOptions(1, 1, [])
network = PytorchNetwork()
network.init(data, network_options, train_options)
network.load(file_name)
loss = network.validate()
os.remove(file_name)
self.assertLess(loss, self.epsilon)
def train(args, data, normalizer, networks):
normalized_data = {
'train': (normalizer.normalize(data['train'][0]), data['train'][1]),
'valid': (normalizer.normalize(data['valid'][0]), data['valid'][1])
}
train_options = TrainOptions(
args.epochs, args.batch_size, args.print_every, args.gpu,
args.optimizer, args.learning_rate, args.activation_function,
args.loss_function, args.num_runs_per_setting, args.dropout_rate,
args.bias, args.seed, args.deterministic,
args.progress_detection_patience, args.progress_detection_min_delta,
args.visualize)
num_features_in = data['train'][0].shape[1]
num_features_out = data['train'][1].shape[1]
network_options = NetworkOptions(num_features_in, num_features_out,
args.size_hidden)
optimizer = Optimizer()
result = optimizer.run(networks, normalized_data, network_options,
train_options, OptimizerOptions(args.model_file))
return result