def __init__(self, dataset_name, model_name, optimizer_name, trial_num): """ :param dataset_name: name of the dataset :type dataset_name: str :param model_name: name of the model :type model_name: str :param optimizer_name: name of the optimizer :type optimizer_name: str :param trial_num: current number of repeated trials :type trial_num: int """ # get optimized hyperparameters with open( f'../params/{dataset_name}_{model_name}_{optimizer_name}/result.json' ) as f: params = json.load(f) # get instances self.dataset = Datasets.get(dataset_name) self.model = Models.get(model_name, dataset=self.dataset) self.optimizer = Optimizers.get(optimizer_name, params=params) # get config with open('./config.json') as f: config = json.load(f) # get constants c = config['constants'][dataset_name][model_name] self.loss = c['loss'] self.batch_size = c['batch_size'] self.epochs = c['epochs'] # configure and initialize directory d = self.main_dir = f'../data/{dataset_name}_{model_name}_{optimizer_name}/trial{trial_num}' if os.path.exists(d): shutil.rmtree(d) os.makedirs(d) # configure hyperdash experiment self.hd_exp = HyperdashExperiment( f'{dataset_name}', api_key_getter=lambda: config['hyperdash']['api_key']) self.hd_exp.param('dataset_name', dataset_name) self.hd_exp.param('model_name', model_name) self.hd_exp.param('optimizer_name', optimizer_name) self.hd_exp.param('trial_num', trial_num) for k, v in params.items(): self.hd_exp.param(k, v) # set callbacks self.callbacks = [ Hyperdash(['accuracy', 'loss', 'val_accuracy', 'val_loss'], self.hd_exp), TensorBoard(log_dir=f'{self.main_dir}/tensorboard'), TimeLogger(filename=f'{self.main_dir}/time.csv'), CSVLogger(filename=f'{self.main_dir}/result.csv', append=True) ]
def objective(self, params): """ objective function to optimize :param params: hyperparamters for optimizer :return: maximum validation accuracy :rtype: float """ # get instances dataset = Datasets.get(self.dataset_name) model = Models.get(self.model_name, dataset=dataset) optimizer = Optimizers.get(self.optimizer_name, params=params) # configure hyperdash experiment hd_exp = HyperdashExperiment( f'{self.dataset_name}', api_key_getter=lambda: self.config['hyperdash']['api_key']) hd_exp.param('dataset_name', self.dataset_name) hd_exp.param('model_name', self.model_name) hd_exp.param('optimizer_name', self.optimizer_name) for k, v in params.items(): hd_exp.param(k, v) # set callbacks callbacks = [ Hyperdash(['accuracy', 'loss', 'val_accuracy', 'val_loss'], hd_exp), EarlyStopping('val_accuracy', patience=10, min_delta=0.01, verbose=1), TerminateOnNaN() ] # get data (x_train, y_train), *_ = dataset.get_batch() # start learning model.compile(loss=self.loss, optimizer=optimizer, metrics=['accuracy']) history = model.fit(x_train, y_train, batch_size=self.batch_size, epochs=self.epochs, callbacks=callbacks, validation_split=0.2, verbose=2) # stop hyperdash experiment hd_exp.end() # return maximum validation accuracy val_accuracy = np.array(history.history['val_accuracy']) return max(val_accuracy) * (-1)
def main(args): """Execute a task based on the given command-line arguments. This function is the main entry-point of the program. It allows the user to extract features, train a model, compute predictions, and evaluate predictions using the command-line interface. """ from datasets import Datasets datasets = Datasets(args.dataset_path) if args.command == 'extract': extract(datasets.get(args.dataset), args) elif args.command == 'train': train(datasets.get('training'), args) elif args.command == 'predict': predict(datasets.get(args.dataset), args) elif args.command == 'evaluate': if isinstance(args.training_id, list): evaluate_all(datasets.get('test'), args) else: evaluate(datasets.get('test'), args)