def _get_trained_network(data, hidden_layer_sizes, train_options=TrainOptions()): default_train_options = TrainOptions(num_epochs=500, optimizer="SGD", learning_rate=0.001, loss_function="MSELoss", print_every=100, use_gpu=True, activation_function="relu", bias=True, dropout_rate=0.5) option_dict = { k: v for k, v in train_options._asdict().items() if v is not None } option_dict.update({ k: None for k, v in train_options._asdict().items() if v == "none" }) train_options = default_train_options._replace(**option_dict) network_options = NetworkOptions(len(data['train'][0][0]), len(data['train'][1][0]), hidden_layer_sizes) network = PytorchNetwork() network.init(data, network_options, train_options) network.train() return network
def run(self, networks: List[Network], data: dict, network_options: NetworkOptions, train_options: TrainOptions, optimizer_options: OptimizerOptions): best = {'loss': None, 'train_options': None, 'network_options': None, 'network': None} for network in networks: logging.info(f"Using {network.__class__.__name__}...") # for all hidden layers all_hidden_layer_sizes = self.hidden_layers if network_options.hidden_layer_sizes is None else \ [network_options.hidden_layer_sizes] for hidden_layer_sizes in all_hidden_layer_sizes: network_options = NetworkOptions(network_options.input_layer_size, network_options.output_layer_size, hidden_layer_sizes) # for all optimizers ... optimizers = self.optimizers if train_options.optimizer is None else [train_options.optimizer] for optimizer in optimizers: optimizer = self.optimizers_map[network.__class__.__name__][optimizer] options = train_options._replace(optimizer=optimizer) # ... and for all loss functions ... loss_functions = self.loss_functions if train_options.loss_function is None else \ [train_options.loss_function] for loss_function in loss_functions: loss_function = self.loss_functions_map[network.__class__.__name__][loss_function] options = options._replace(loss_function=loss_function) # ... run x times (try several times because of the random seed) num_runs = train_options.num_runs_per_setting \ if train_options.num_runs_per_setting is not None else self.default_num_runs_per_setting for i in range(1, num_runs + 1): self._run_once(best, options, data, i, network, network_options, optimizer_options) return best
def test_run_no_hidden_layers_specified_should_enumerate_over_all_hidden_layers(self): network_mock = Mock() network_mock.__class__.__name__ = "PytorchNetwork" network_mock.validate.return_value = 1 optimizer = Optimizer() optimizer.run([network_mock], {}, NetworkOptions(0, 0, None), TrainOptions(optimizer="adam", loss_function="mse"), OptimizerOptions(None)) network_mock.init.assert_called() self.assertEqual(optimizer.default_num_runs_per_setting * len(optimizer.hidden_layers), network_mock.init.call_count)
def test_run_should_call_network_multiple_times(self): network_mock = Mock() network_mock.__class__.__name__ = "PytorchNetwork" network_mock.validate.return_value = 1 optimizer = Optimizer() optimizer.run([network_mock], {}, NetworkOptions(0, 0, []), TrainOptions(optimizer="adam", loss_function="mse"), OptimizerOptions(None)) network_mock.init.assert_called() self.assertEqual(optimizer.default_num_runs_per_setting, network_mock.init.call_count)
def test_run_no_optional_options_specified_using_linear_data(self): data = self._get_data_linear() train_options = TrainOptions(print_every=100, use_gpu=True, activation_function="relu", num_epochs=100) network_options = NetworkOptions(len(data['train'][0][0]), len(data['train'][1][0]), None) network = PytorchNetwork() optimizer = Optimizer() best = optimizer.run([network], data, network_options, train_options, OptimizerOptions(None)) logging.info(f"Best run: {best}") self.assertLess(best['loss'], 100) self.assertIsNotNone(best['train_options'])
def test_load(self): network = self._get_trained_network(self._get_data_linear(), [], self._get_train_options_linear()) file_name = "tmp.pth" network.save(file_name) data = self._get_data_linear() train_options = TrainOptions() network_options = NetworkOptions(1, 1, []) network = PytorchNetwork() network.init(data, network_options, train_options) network.load(file_name) loss = network.validate() os.remove(file_name) self.assertLess(loss, self.epsilon)
def train(args, data, normalizer, networks): normalized_data = { 'train': (normalizer.normalize(data['train'][0]), data['train'][1]), 'valid': (normalizer.normalize(data['valid'][0]), data['valid'][1]) } train_options = TrainOptions( args.epochs, args.batch_size, args.print_every, args.gpu, args.optimizer, args.learning_rate, args.activation_function, args.loss_function, args.num_runs_per_setting, args.dropout_rate, args.bias, args.seed, args.deterministic, args.progress_detection_patience, args.progress_detection_min_delta, args.visualize) num_features_in = data['train'][0].shape[1] num_features_out = data['train'][1].shape[1] network_options = NetworkOptions(num_features_in, num_features_out, args.size_hidden) optimizer = Optimizer() result = optimizer.run(networks, normalized_data, network_options, train_options, OptimizerOptions(args.model_file)) return result