default=20, help="number of trials to run (defaults to 20)", ) if __name__ == "__main__": args = parser.parse_args() # Main loop: for i in range(args.num_trials): run_trial() print("Summary of run {}/{}:".format(i + 1, args.num_trials)) pprint(bb.get_current_run()) print() print("\nSummary of best run:") pprint(bb.get_optimal_run()) print("Displaying plots...") fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, figsize=(10, 10)) bb.plot_convergence(ax1) bb.plot_history(ax2) bb.plot_partial_dependence_1D("hidden neurons", ax3) bb.plot_partial_dependence_1D("learning rate", ax4) plt.figure(1) bb.plot_evaluations() plt.figure(2) bb.plot_objective()
class TrainOptimizeScript(TrainScript): Usage(''' Use black box optimization to tune model hyperparameters :-t --trials-name str - Filename to save hyperparameter optimization trials in '.bbopt.json' will automatically be appended :-c --cycles int 20 Number of cycles of optimization to run :-m --model str .cache/optimized.net Model to load from ... ''') | TrainScript.usage def __init__(self, args): super().__init__(args) from bbopt import BlackBoxOptimizer self.bb = BlackBoxOptimizer(file=self.args.trials_name) if not self.test: data = TrainData.from_both(self.args.tags_file, self.args.tags_folder, self.args.folder) _, self.test = data.load(False, True) from keras.callbacks import ModelCheckpoint for i in list(self.callbacks): if isinstance(i, ModelCheckpoint): self.callbacks.remove(i) def process_args(self, args: Any): model_parts = glob(splitext(args.model)[0] + '.*') if len(model_parts) < 5: for name in model_parts: if isfile(name): remove(name) else: rmtree(name) args.trials_name = args.trials_name.replace('.bbopt.json', '').replace('.json', '') if not args.trials_name: if isfile(join('.cache', 'trials.bbopt.json')): remove(join('.cache', 'trials.bbopt.json')) args.trials_name = join('.cache', 'trials') def run(self): print('Writing to:', self.args.trials_name + '.bbopt.json') for i in range(self.args.cycles): self.bb.run(backend="random") print("\n= %d = (example #%d)" % (i + 1, len(self.bb.get_data()["examples"]) + 1)) params = ModelParams(recurrent_units=self.bb.randint("units", 1, 70, guess=50), dropout=self.bb.uniform("dropout", 0.1, 0.9, guess=0.6), extra_metrics=self.args.extra_metrics, skip_acc=self.args.no_validation, loss_bias=1.0 - self.args.sensitivity) print('Testing with:', params) model = create_model(self.args.model, params) model.fit(*self.sampled_data, batch_size=self.args.batch_size, epochs=self.epoch + self.args.epochs, validation_data=self.test * (not self.args.no_validation), callbacks=self.callbacks, initial_epoch=self.epoch) resp = model.evaluate(*self.test, batch_size=self.args.batch_size) if not isinstance(resp, (list, tuple)): resp = [resp, None] test_loss, test_acc = resp predictions = model.predict(self.test[0], batch_size=self.args.batch_size) num_false_positive = numpy.sum(predictions * (1 - self.test[1]) > 0.5) num_false_negative = numpy.sum( (1 - predictions) * self.test[1] > 0.5) false_positives = num_false_positive / numpy.sum( self.test[1] < 0.5) false_negatives = num_false_negative / numpy.sum( self.test[1] > 0.5) from math import exp param_score = 1.0 / (1.0 + exp( (model.count_params() - 11000) / 2000)) fitness = param_score * (1.0 - 0.2 * false_negatives - 0.8 * false_positives) self.bb.remember({ "test loss": test_loss, "test accuracy": test_acc, "false positive%": false_positives, "false negative%": false_negatives, "fitness": fitness }) print("False positive: ", false_positives * 100, "%") self.bb.maximize(fitness) pprint(self.bb.get_current_run()) best_example = self.bb.get_optimal_run() print("\n= BEST = (example #%d)" % self.bb.get_data()["examples"].index(best_example)) pprint(best_example)