Exemple #1
0
    def execute(self, no_epoch, no_update):
        savedir = pjoin(self.savedir, str(no_epoch))
        if not os.path.isdir():
            os.makedirs(savedir)

        self.model.save(pjoin(savedir, "model.pkl"))
        status = {"no_epoch": no_epoch, "no_update": no_update}
        utils.save_dict_to_json_file(pjoin(savedir, "status.json"), status)
Exemple #2
0
 def execute(self, no_epoch, no_update):
     self.model.save(pjoin(self.savedir, "model.pkl"))
     status = {
         'no_epoch': no_epoch,
         'no_update': no_update,
         'hidden_size': self.model.hidden_size
     }
     utils.save_dict_to_json_file(pjoin(self.savedir, "status.json"),
                                  status)
Exemple #3
0
    def execute(self, no_epoch, no_update):
        savedir = pjoin(self.savedir, str(no_epoch))
        if not os.path.isdir(savedir):
            os.makedirs(savedir)

        shutil.copy(pjoin(self.savedir, "hyperparams.json"), pjoin(savedir, "hyperparams.json"))

        self.model.save(pjoin(savedir, "model.pkl"))
        status = {"no_epoch": no_epoch, "no_update": no_update, "hidden_size": self.model.hidden_size}
        utils.save_dict_to_json_file(pjoin(savedir, "status.json"), status)
Exemple #4
0
    def execute(self, no_epoch, no_update):
        savedir = pjoin(self.savedir, str(no_epoch))
        if not os.path.isdir(savedir):
            os.makedirs(savedir)

        shutil.copy(pjoin(self.savedir, 'hyperparams.json'),
                    pjoin(savedir, 'hyperparams.json'))

        self.model.save(pjoin(savedir, "model.pkl"))
        status = {
            'no_epoch': no_epoch,
            'no_update': no_update,
            'hidden_size': self.model.hidden_size
        }
        utils.save_dict_to_json_file(pjoin(savedir, "status.json"), status)
Exemple #5
0
def main():
    parser = buildArgsParser()
    args = parser.parse_args()

    # Get experiment folder
    experiment_path = args.name
    if not os.path.isdir(experiment_path):
        # If not a directory, it must be the name of the experiment.
        experiment_path = pjoin(".", "experiments", args.name)

    if not os.path.isdir(experiment_path):
        parser.error('Cannot find experiment: {0}!'.format(args.name))

    if not os.path.isfile(pjoin(experiment_path, "model.pkl")):
        parser.error(
            'Cannot find model for experiment: {0}!'.format(experiment_path))

    if not os.path.isfile(pjoin(experiment_path, "hyperparams.json")):
        parser.error('Cannot find hyperparams for experiment: {0}!'.format(
            experiment_path))

    # Load experiments hyperparameters
    hyperparams = utils.load_dict_from_json_file(
        pjoin(experiment_path, "hyperparams.json"))

    with Timer("Loading dataset"):
        trainset, validset, testset = dataset.load(
            hyperparams['dataset'], hyperparams.get('dataset_percent', 1.))
        print " (data: {:,}; {:,}; {:,}) ".format(len(trainset), len(validset),
                                                  len(testset)),

    with Timer("Loading model"):
        if hyperparams["model"] == "rbm":
            from iRBM.models.rbm import RBM
            model_class = RBM
        elif hyperparams["model"] == "orbm":
            from iRBM.models.orbm import oRBM
            model_class = oRBM
        elif hyperparams["model"] == "irbm":
            from iRBM.models.irbm import iRBM
            model_class = iRBM

        # Load the actual model.
        model = model_class.load(pjoin(experiment_path, "model.pkl"))

        if args.irbm_fixed_size:
            # Use methods from the oRBM.
            import functools
            from iRBM.models.orbm import oRBM
            setattr(model, "get_base_rate",
                    functools.partial(oRBM.get_base_rate, model))
            setattr(model, "pdf_z_given_v",
                    functools.partial(oRBM.pdf_z_given_v, model))
            setattr(model, "log_z_given_v",
                    functools.partial(oRBM.log_z_given_v, model))
            setattr(model, "free_energy",
                    functools.partial(oRBM.free_energy, model))
            print "({} with {} fixed hidden units)".format(
                hyperparams["model"], model.hidden_size)

        else:
            print "({} with {} hidden units)".format(hyperparams["model"],
                                                     model.hidden_size)

    # Result files.
    if args.irbm_fixed_size:
        experiment_path = pjoin(experiment_path, "irbm_fixed_size")
        try:
            os.makedirs(experiment_path)
        except:
            pass

    ais_result_file = pjoin(experiment_path, "ais_result.json")
    result_file = pjoin(experiment_path, "result.json")

    if args.lnZ is not None:
        lnZ, lnZ_down, lnZ_up = args.lnZ
    else:
        if not os.path.isfile(ais_result_file) or args.force:
            with Timer(
                    "Estimating model's partition function with AIS({0}) and {1} temperatures."
                    .format(args.nb_samples, args.nb_temperatures)):
                ais_results = compute_AIS(model,
                                          M=args.nb_samples,
                                          betas=np.linspace(
                                              0, 1, args.nb_temperatures),
                                          seed=args.seed,
                                          ais_working_dir=experiment_path,
                                          force=args.force)
                ais_results["irbm_fixed_size"] = args.irbm_fixed_size
                utils.save_dict_to_json_file(ais_result_file, ais_results)
        else:
            print "Loading previous AIS results... (use --force to re-run AIS)"
            ais_results = utils.load_dict_from_json_file(ais_result_file)
            print "AIS({0}) with {1} temperatures".format(
                ais_results['nb_samples'], ais_results['nb_temperatures'])

            if ais_results['nb_samples'] != args.nb_samples:
                print "The number of samples specified ({:,}) doesn't match the one found in ais_results.json ({:,}). Aborting.".format(
                    args.nb_samples, ais_results['nb_samples'])
                sys.exit(-1)

            if ais_results['nb_temperatures'] != args.nb_temperatures:
                print "The number of temperatures specified ({:,}) doesn't match the one found in ais_results.json ({:,}). Aborting.".format(
                    args.nb_temperatures, ais_results['nb_temperatures'])
                sys.exit(-1)

            if ais_results['seed'] != args.seed:
                print "The seed specified ({}) doesn't match the one found in ais_results.json ({}). Aborting.".format(
                    args.seed, ais_results['seed'])
                sys.exit(-1)

            if ais_results.get('irbm_fixed_size',
                               False) != args.irbm_fixed_size:
                print "The option '--irbm-fixed' specified ({}) doesn't match the one found in ais_results.json ({}). Aborting.".format(
                    args.irbm_fixed_size, ais_results['irbm_fixed_size'])
                sys.exit(-1)

        lnZ = ais_results['logcummean_Z'][-1]
        logcumstd_Z_down = ais_results['logcumstd_Z_down'][-1]
        logcumstd_Z_up = ais_results['logcumstd_Z_up'][-1]
        lnZ_down = lnZ - logcumstd_Z_down
        lnZ_up = lnZ + logcumstd_Z_up

    print "-> lnZ: {lnZ_down} <= {lnZ} <= {lnZ_up}".format(lnZ_down=lnZ_down,
                                                           lnZ=lnZ,
                                                           lnZ_up=lnZ_up)

    with Timer("\nComputing average NLL on {0} using lnZ={1}.".format(
            hyperparams['dataset'], lnZ)):
        NLL_train, NLL_valid, NLL_test = compute_AvgStderrNLL(
            model, lnZ, trainset, validset, testset)

    print "Avg. NLL on trainset: {:.2f} ± {:.2f}".format(
        NLL_train.avg, NLL_train.stderr)
    print "Avg. NLL on validset: {:.2f} ± {:.2f}".format(
        NLL_valid.avg, NLL_valid.stderr)
    print "Avg. NLL on testset:  {:.2f} ± {:.2f}".format(
        NLL_test.avg, NLL_test.stderr)

    # Save results JSON file.
    if args.lnZ is None:
        result = {
            'lnZ': float(lnZ),
            'lnZ_down': float(lnZ_down),
            'lnZ_up': float(lnZ_up),
            'trainset': [float(NLL_train.avg),
                         float(NLL_train.stderr)],
            'validset': [float(NLL_valid.avg),
                         float(NLL_valid.stderr)],
            'testset': [float(NLL_test.avg),
                        float(NLL_test.stderr)],
            'irbm_fixed_size': args.irbm_fixed_size,
        }
        utils.save_dict_to_json_file(result_file, result)

    if args.view:

        from iRBM.misc import vizu
        import matplotlib.pyplot as plt

        if hyperparams["dataset"] == "binarized_mnist":
            image_shape = (28, 28)
        elif hyperparams["dataset"] == "caltech101_silhouettes28":
            image_shape = (28, 28)
        else:
            raise ValueError("Unknown dataset: {0}".format(
                hyperparams["dataset"]))

        # Display AIS samples.
        data = vizu.concatenate_images(ais_results['last_sample_chain'],
                                       shape=image_shape,
                                       border_size=1,
                                       clim=(0, 1))
        plt.figure()
        plt.imshow(data, cmap=plt.cm.gray, interpolation='nearest')
        plt.title("AIS samples")

        # Display AIS ~lnZ.
        plt.figure()
        plt.gca().set_xmargin(0.1)
        plt.errorbar(np.arange(ais_results['nb_samples']) + 1,
                     ais_results["logcummean_Z"],
                     yerr=[
                         ais_results['logcumstd_Z_down'],
                         ais_results['logcumstd_Z_up']
                     ],
                     fmt='ob',
                     label='with std ~ln std Z')
        plt.legend()
        plt.ticklabel_format(useOffset=False, axis='y')
        plt.title("~ln mean Z for different number of AIS samples")
        plt.ylabel("~lnZ")
        plt.xlabel("# AIS samples")

        plt.show()
Exemple #6
0
def main():
    parser = buildArgsParser()
    args = parser.parse_args()

    # Get experiment folder
    experiment_path = args.name
    if not os.path.isdir(experiment_path):
        # If not a directory, it must be the name of the experiment.
        experiment_path = pjoin(".", "experiments", args.name)

    if not os.path.isdir(experiment_path):
        parser.error('Cannot find experiment: {0}!'.format(args.name))

    if not os.path.isfile(pjoin(experiment_path, "model.pkl")):
        parser.error('Cannot find model for experiment: {0}!'.format(experiment_path))

    if not os.path.isfile(pjoin(experiment_path, "hyperparams.json")):
        parser.error('Cannot find hyperparams for experiment: {0}!'.format(experiment_path))

    # Load experiments hyperparameters
    hyperparams = utils.load_dict_from_json_file(pjoin(experiment_path, "hyperparams.json"))

    with Timer("Loading dataset"):
        trainset, validset, testset = dataset.load(hyperparams['dataset'], hyperparams.get('dataset_percent', 1.))
        print " (data: {:,}; {:,}; {:,}) ".format(len(trainset), len(validset), len(testset)),

    with Timer("Loading model"):
        if hyperparams["model"] == "rbm":
            from iRBM.models.rbm import RBM
            model_class = RBM
        elif hyperparams["model"] == "orbm":
            from iRBM.models.orbm import oRBM
            model_class = oRBM
        elif hyperparams["model"] == "irbm":
            from iRBM.models.irbm import iRBM
            model_class = iRBM

        # Load the actual model.
        model = model_class.load(pjoin(experiment_path, "model.pkl"))

    if args.lnZ is None:
        with Timer("Estimating model's partition function with AIS({0}) and {1} temperatures.".format(args.nb_samples, args.nb_temperatures)):
            lnZ, lnZ_down, lnZ_up = compute_lnZ(model, nb_chains=args.nb_samples, temperatures=np.linspace(0, 1, args.nb_temperatures))
            lnZ_down = lnZ - lnZ_down
            lnZ_up = lnZ + lnZ_up
    else:
        lnZ, lnZ_down, lnZ_up = args.lnZ

    print "-> lnZ: {lnZ_down} <= {lnZ} <= {lnZ_up}".format(lnZ_down=lnZ_down, lnZ=lnZ, lnZ_up=lnZ_up)

    with Timer("\nComputing average NLL on {0} using lnZ={1}.".format(hyperparams['dataset'], lnZ)):
        NLL_train, NLL_valid, NLL_test = compute_AvgStderrNLL(model, lnZ, trainset, validset, testset)

    print "Avg. NLL on trainset: {:.2f} ± {:.2f}".format(NLL_train.avg, NLL_train.stderr)
    print "Avg. NLL on validset: {:.2f} ± {:.2f}".format(NLL_valid.avg, NLL_valid.stderr)
    print "Avg. NLL on testset:  {:.2f} ± {:.2f}".format(NLL_test.avg, NLL_test.stderr)

    # Save results JSON file.
    if not os.path.isfile(pjoin(experiment_path, "result.json")) or args.force:
        result = {'lnZ': lnZ,
                  'lnZ_down': lnZ_down,
                  'lnZ_up': lnZ_up,
                  'trainset': [NLL_train.avg, NLL_train.stderr],
                  'validset': [NLL_valid.avg, NLL_valid.stderr],
                  'testset': [NLL_test.avg, NLL_test.stderr],
                  }
        utils.save_dict_to_json_file(pjoin(experiment_path, "result.json"), result)
Exemple #7
0
def main():
    parser = buildArgsParser()
    args = parser.parse_args()

    # Get experiment folder
    experiment_path = args.name
    if not os.path.isdir(experiment_path):
        # If not a directory, it must be the name of the experiment.
        experiment_path = pjoin(".", "experiments", args.name)

    if not os.path.isdir(experiment_path):
        parser.error('Cannot find experiment: {0}!'.format(args.name))

    if not os.path.isfile(pjoin(experiment_path, "model.pkl")):
        parser.error('Cannot find model for experiment: {0}!'.format(experiment_path))

    if not os.path.isfile(pjoin(experiment_path, "hyperparams.json")):
        parser.error('Cannot find hyperparams for experiment: {0}!'.format(experiment_path))

    # Load experiments hyperparameters
    hyperparams = utils.load_dict_from_json_file(pjoin(experiment_path, "hyperparams.json"))

    with Timer("Loading dataset"):
        trainset, validset, testset = dataset.load(hyperparams['dataset'], hyperparams.get('dataset_percent', 1.))
        print " (data: {:,}; {:,}; {:,}) ".format(len(trainset), len(validset), len(testset)),

    with Timer("Loading model"):
        if hyperparams["model"] == "rbm":
            from iRBM.models.rbm import RBM
            model_class = RBM
        elif hyperparams["model"] == "orbm":
            from iRBM.models.orbm import oRBM
            model_class = oRBM
        elif hyperparams["model"] == "irbm":
            from iRBM.models.irbm import iRBM
            model_class = iRBM

        # Load the actual model.
        model = model_class.load(pjoin(experiment_path, "model.pkl"))

        if args.irbm_fixed_size:
            # Use methods from the oRBM.
            import functools
            from iRBM.models.orbm import oRBM
            setattr(model, "get_base_rate", functools.partial(oRBM.get_base_rate, model))
            setattr(model, "pdf_z_given_v", functools.partial(oRBM.pdf_z_given_v, model))
            setattr(model, "log_z_given_v", functools.partial(oRBM.log_z_given_v, model))
            setattr(model, "free_energy", functools.partial(oRBM.free_energy, model))
            print "({} with {} fixed hidden units)".format(hyperparams["model"], model.hidden_size)

        else:
            print "({} with {} hidden units)".format(hyperparams["model"], model.hidden_size)

    # Result files.
    if args.irbm_fixed_size:
        experiment_path = pjoin(experiment_path, "irbm_fixed_size")
        try:
            os.makedirs(experiment_path)
        except:
            pass

    ais_result_file = pjoin(experiment_path, "ais_result.json")
    result_file = pjoin(experiment_path, "result.json")

    if args.lnZ is not None:
        lnZ, lnZ_down, lnZ_up = args.lnZ
    else:
        if not os.path.isfile(ais_result_file) or args.force:
            with Timer("Estimating model's partition function with AIS({0}) and {1:,} temperatures.".format(args.nb_samples, args.nb_temperatures)):
                ais_results = compute_AIS(model, M=args.nb_samples, betas=np.linspace(0, 1, args.nb_temperatures), seed=args.seed, ais_working_dir=experiment_path, force=args.force)
                ais_results["irbm_fixed_size"] = args.irbm_fixed_size
                utils.save_dict_to_json_file(ais_result_file, ais_results)
        else:
            print "Loading previous AIS results... (use --force to re-run AIS)"
            ais_results = utils.load_dict_from_json_file(ais_result_file)
            print "AIS({0}) with {1:,} temperatures".format(ais_results['nb_samples'], ais_results['nb_temperatures'])

            if ais_results['nb_samples'] != args.nb_samples:
                print "The number of samples specified ({:,}) doesn't match the one found in ais_results.json ({:,}). Aborting.".format(args.nb_samples, ais_results['nb_samples'])
                sys.exit(-1)

            if ais_results['nb_temperatures'] != args.nb_temperatures:
                print "The number of temperatures specified ({:,}) doesn't match the one found in ais_results.json ({:,}). Aborting.".format(args.nb_temperatures, ais_results['nb_temperatures'])
                sys.exit(-1)

            if ais_results['seed'] != args.seed:
                print "The seed specified ({}) doesn't match the one found in ais_results.json ({}). Aborting.".format(args.seed, ais_results['seed'])
                sys.exit(-1)

            if ais_results.get('irbm_fixed_size', False) != args.irbm_fixed_size:
                print "The option '--irbm-fixed' specified ({}) doesn't match the one found in ais_results.json ({}). Aborting.".format(args.irbm_fixed_size, ais_results['irbm_fixed_size'])
                sys.exit(-1)

        lnZ = ais_results['logcummean_Z'][-1]
        logcumstd_Z_down = ais_results['logcumstd_Z_down'][-1]
        logcumstd_Z_up = ais_results['logcumstd_Z_up'][-1]
        lnZ_down = lnZ - logcumstd_Z_down
        lnZ_up = lnZ + logcumstd_Z_up

    print "-> lnZ: {lnZ_down} <= {lnZ} <= {lnZ_up}".format(lnZ_down=lnZ_down, lnZ=lnZ, lnZ_up=lnZ_up)

    with Timer("\nComputing average NLL on {0} using lnZ={1}.".format(hyperparams['dataset'], lnZ)):
        NLL_train, NLL_valid, NLL_test = compute_AvgStderrNLL(model, lnZ, trainset, validset, testset)

    print "Avg. NLL on trainset: {:.2f} ± {:.2f}".format(NLL_train.avg, NLL_train.stderr)
    print "Avg. NLL on validset: {:.2f} ± {:.2f}".format(NLL_valid.avg, NLL_valid.stderr)
    print "Avg. NLL on testset:  {:.2f} ± {:.2f}".format(NLL_test.avg, NLL_test.stderr)
    print "---"
    Fv_rnd = model.free_energy(np.random.rand(*ais_results['last_sample_chain'].shape)).eval()
    print "Avg. F(v) on {:,} random samples: {:.2f} ± {:.2f}".format(args.nb_samples, Fv_rnd.mean(), Fv_rnd.std())
    Fv_model = model.free_energy(ais_results['last_sample_chain']).eval()
    print "Avg. F(v) on {:,} AIS samples:    {:.2f} ± {:.2f}".format(args.nb_samples, Fv_model.mean(), Fv_model.std())

    # Save results JSON file.
    if args.lnZ is None:
        result = {'lnZ': float(lnZ),
                  'lnZ_down': float(lnZ_down),
                  'lnZ_up': float(lnZ_up),
                  'trainset': [float(NLL_train.avg), float(NLL_train.stderr)],
                  'validset': [float(NLL_valid.avg), float(NLL_valid.stderr)],
                  'testset': [float(NLL_test.avg), float(NLL_test.stderr)],
                  'irbm_fixed_size': args.irbm_fixed_size,
                  }
        utils.save_dict_to_json_file(result_file, result)

    if args.view:

        from iRBM.misc import vizu
        import matplotlib.pyplot as plt

        if hyperparams["dataset"] == "binarized_mnist":
            image_shape = (28, 28)
        elif hyperparams["dataset"] == "caltech101_silhouettes28":
            image_shape = (28, 28)
        else:
            raise ValueError("Unknown dataset: {0}".format(hyperparams["dataset"]))

        # Display AIS samples.
        data = vizu.concatenate_images(ais_results['last_sample_chain'], shape=image_shape, border_size=1, clim=(0, 1))
        plt.figure()
        plt.imshow(data, cmap=plt.cm.gray, interpolation='nearest')
        plt.title("AIS samples")

        # Display AIS ~lnZ.
        plt.figure()
        plt.gca().set_xmargin(0.1)
        plt.errorbar(np.arange(ais_results['nb_samples'])+1, ais_results["logcummean_Z"],
                     yerr=[ais_results['logcumstd_Z_down'], ais_results['logcumstd_Z_up']],
                     fmt='ob', label='with std ~ln std Z')
        plt.legend()
        plt.ticklabel_format(useOffset=False, axis='y')
        plt.title("~ln mean Z for different number of AIS samples")
        plt.ylabel("~lnZ")
        plt.xlabel("# AIS samples")

        plt.show()
Exemple #8
0
 def execute(self, no_epoch, no_update):
     self.model.save(pjoin(self.savedir, "model.pkl"))
     status = {"no_epoch": no_epoch, "no_update": no_update}
     utils.save_dict_to_json_file(pjoin(self.savedir, "status.json"), status)
Exemple #9
0
def main():
    parser = buildArgsParser()
    args = parser.parse_args()

    # Extract experiments hyperparameters
    hyperparams = dict(vars(args))
    # Remove hyperparams that should not be part of the hash
    del hyperparams['nb_epochs']
    del hyperparams['max_epoch']
    del hyperparams['keep']
    del hyperparams['force']
    del hyperparams['name']

    # Get/generate experiment name
    experiment_name = args.name
    if experiment_name is None:
        experiment_name = utils.generate_uid_from_string(repr(hyperparams))

    # Create experiment folder
    experiment_path = pjoin(".", "experiments", experiment_name)
    resuming = False
    if os.path.isdir(experiment_path) and not args.force:
        resuming = True
        print "### Resuming experiment ({0}). ###\n".format(experiment_name)
        # Check if provided hyperparams match those in the experiment folder
        hyperparams_loaded = utils.load_dict_from_json_file(
            pjoin(experiment_path, "hyperparams.json"))
        if hyperparams != hyperparams_loaded:
            print "The arguments provided are different than the one saved. Use --force if you are certain.\nQuitting."
            exit(1)
    else:
        if os.path.isdir(experiment_path):
            shutil.rmtree(experiment_path)

        os.makedirs(experiment_path)
        utils.save_dict_to_json_file(
            pjoin(experiment_path, "hyperparams.json"), hyperparams)

    with Timer("Loading dataset"):
        trainset, validset, testset = dataset.load(args.dataset,
                                                   args.dataset_percent)
        print " (data: {:,}; {:,}; {:,}) ".format(len(trainset), len(validset),
                                                  len(testset)),

    with Timer("\nCreating model"):
        model = model_factory(args.model,
                              input_size=trainset.input_size,
                              hyperparams=hyperparams)

    starting_epoch = 1
    if resuming:
        with Timer("\nLoading model"):
            status = utils.load_dict_from_json_file(
                pjoin(experiment_path, "status.json"))
            starting_epoch = status['no_epoch'] + 1
            model = model.load(pjoin(experiment_path, "model.pkl"))

    ### Build trainer ###
    with Timer("\nBuilding trainer"):
        trainer = Trainer(model,
                          trainset,
                          batch_size=hyperparams['batch_size'],
                          starting_epoch=starting_epoch)

        # Add stopping criteria
        ending_epoch = args.max_epoch if args.max_epoch is not None else starting_epoch + args.nb_epochs - 1
        # Stop when max number of epochs is reached.
        trainer.add_stopping_criterion(tasks.MaxEpochStopping(ending_epoch))

        # Print time a training epoch took
        trainer.add_task(tasks.PrintEpochDuration())
        avg_reconstruction_error = tasks.AverageReconstructionError(
            model.CD.chain_start, model.CD.chain_end, len(trainset))
        trainer.add_task(
            tasks.Print(avg_reconstruction_error,
                        msg="Avg. reconstruction error: {0:.1f}"))

        if args.model == 'irbm':
            trainer.add_task(
                irbm.GrowiRBM(model,
                              shrinkable=args.shrinkable,
                              nb_neurons_to_add=args.nb_neurons_to_add))

        # Save training progression
        trainer.add_task(
            tasks.SaveProgression(model, experiment_path, each_epoch=50))
        if args.keep is not None:
            trainer.add_task(
                tasks.KeepProgression(model,
                                      experiment_path,
                                      each_epoch=args.keep))

        trainer.build()

    print "\nWill train {0} from epoch {1} to epoch {2}.".format(
        args.model, starting_epoch, ending_epoch)
    trainer.train()

    with Timer("\nSaving"):
        # Save final model
        model.save(pjoin(experiment_path, "model.pkl"))
Exemple #10
0
def main():
    parser = buildArgsParser()
    args = parser.parse_args()

    # Extract experiments hyperparameters
    hyperparams = dict(vars(args))
    # Remove hyperparams that should not be part of the hash
    del hyperparams['nb_epochs']
    del hyperparams['max_epoch']
    del hyperparams['keep']
    del hyperparams['force']
    del hyperparams['name']

    # Get/generate experiment name
    experiment_name = args.name
    if experiment_name is None:
        experiment_name = utils.generate_uid_from_string(repr(hyperparams))

    # Create experiment folder
    experiment_path = pjoin(".", "experiments", experiment_name)
    resuming = False
    if os.path.isdir(experiment_path) and not args.force:
        resuming = True
        print "### Resuming experiment ({0}). ###\n".format(experiment_name)
        # Check if provided hyperparams match those in the experiment folder
        hyperparams_loaded = utils.load_dict_from_json_file(pjoin(experiment_path, "hyperparams.json"))
        if hyperparams != hyperparams_loaded:
            print "The arguments provided are different than the one saved. Use --force if you are certain.\nQuitting."
            exit(1)
    else:
        if os.path.isdir(experiment_path):
            shutil.rmtree(experiment_path)

        os.makedirs(experiment_path)
        utils.save_dict_to_json_file(pjoin(experiment_path, "hyperparams.json"), hyperparams)

    with Timer("Loading dataset"):
        trainset, validset, testset = dataset.load(args.dataset, args.dataset_percent)
        print " (data: {:,}; {:,}; {:,}) ".format(len(trainset), len(validset), len(testset)),

    with Timer("\nCreating model"):
        model = model_factory(args.model, input_size=trainset.input_size, hyperparams=hyperparams)

    starting_epoch = 1
    if resuming:
        with Timer("\nLoading model"):
            status = utils.load_dict_from_json_file(pjoin(experiment_path, "status.json"))
            starting_epoch = status['no_epoch'] + 1
            model = model.load(pjoin(experiment_path, "model.pkl"))

    ### Build trainer ###
    with Timer("\nBuilding trainer"):
        trainer = Trainer(model, trainset, batch_size=hyperparams['batch_size'], starting_epoch=starting_epoch)

        # Add stopping criteria
        ending_epoch = args.max_epoch if args.max_epoch is not None else starting_epoch + args.nb_epochs - 1
        # Stop when max number of epochs is reached.
        trainer.add_stopping_criterion(tasks.MaxEpochStopping(ending_epoch))

        # Print time a training epoch took
        trainer.add_task(tasks.PrintEpochDuration())
        avg_reconstruction_error = tasks.AverageReconstructionError(model.CD.chain_start, model.CD.chain_end, len(trainset))
        trainer.add_task(tasks.Print(avg_reconstruction_error, msg="Avg. reconstruction error: {0:.1f}"))

        if args.model == 'irbm':
            trainer.add_task(irbm.GrowiRBM(model, shrinkable=args.shrinkable, nb_neurons_to_add=args.nb_neurons_to_add))

        # Save training progression
        trainer.add_task(tasks.SaveProgression(model, experiment_path, each_epoch=50))
        if args.keep is not None:
            trainer.add_task(tasks.KeepProgression(model, experiment_path, each_epoch=args.keep))

        trainer.build()

    print "\nWill train {0} from epoch {1} to epoch {2}.".format(args.model, starting_epoch, ending_epoch)
    trainer.train()

    with Timer("\nSaving"):
        # Save final model
        model.save(pjoin(experiment_path, "model.pkl"))
def _compute_AIS(model, M=100, betas=BETAS, batch_size=None, seed=1234, ais_working_dir=".", force=False):
    ais_results_json = pjoin(ais_working_dir, "ais_results.part.json")

    if batch_size is None:
        batch_size = M

    # Will be executing M AIS's runs.
    last_sample_chain = np.zeros((M, model.input_size), dtype=config.floatX)
    M_log_w_ais = np.zeros(M, dtype=np.float64)

    model.set_rng_seed(seed)

    ais_results = {}
    if os.path.isfile(ais_results_json) and not force:
        print "Resuming AIS using info from {}".format(ais_results_json)
        ais_results = utils.load_dict_from_json_file(ais_results_json)
        M_log_w_ais = ais_results['M_log_w_ais']
        last_sample_chain = ais_results['last_sample_chain']
        lnZ_trivial = ais_results['lnZ_trivial']

    # Iterate through all AIS runs.
    for i in range(0, M, batch_size):
        if i <= ais_results.get('batch_id', -1):
            continue

        model.set_rng_seed(seed+i)
        actual_size = min(M - i, batch_size)
        print "AIS run: {}/{} (using batch size of {})".format(i, M, batch_size)
        ais_partial_results = _compute_AIS_samples(model, M=actual_size, betas=betas)

        M_log_w_ais[i:i+batch_size] = ais_partial_results['M_log_w_ais']
        last_sample_chain[i:i+batch_size] = ais_partial_results['last_sample_chain']
        lnZ_trivial = ais_partial_results['lnZ_trivial']

        # Save partial results
        if os.path.isfile(ais_results_json):
            shutil.copy(ais_results_json, ais_results_json[:-4] + "old.json")

        ais_results = {'batch_id': i,
                       'M': M,
                       'batch_size': batch_size,
                       'last_sample_chain': last_sample_chain,
                       'M_log_w_ais': M_log_w_ais,
                       'lnZ_trivial': lnZ_trivial}
        utils.save_dict_to_json_file(ais_results_json, ais_results)

    # We compute the mean of the estimated `r_AIS`
    Ms = np.arange(1, M+1)
    log_sum_w_ais = np.logaddexp.accumulate(M_log_w_ais)
    logcummean_Z = log_sum_w_ais - np.log(Ms)

    # We compute the standard deviation of the estimated `r_AIS`
    logstd_AIS = np.zeros_like(M_log_w_ais)
    for k in Ms[1:]:
        m = np.max(M_log_w_ais[:k])
        logstd_AIS[k-1] = np.log(np.std(np.exp(M_log_w_ais[:k]-m), ddof=1)) - np.log(np.sqrt(k))
        logstd_AIS[k-1] += m

    logstd_AIS[0] = np.nan  # Standard deviation of only one sample does not exist.

    # The authors report AIS error using ln(Ẑ ± 3\sigma)
    m = max(np.nanmax(logstd_AIS), np.nanmax(logcummean_Z))
    logcumstd_Z_up = np.log(np.exp(logcummean_Z-m) + 3*np.exp(logstd_AIS-m)) + m - logcummean_Z
    logcumstd_Z_down = -(np.log(np.exp(logcummean_Z-m) - 3*np.exp(logstd_AIS-m)) + m) + logcummean_Z

    # Compute the standard deviation of ln(Z)
    std_lnZ = np.array([np.std(M_log_w_ais[:k], ddof=1) for k in Ms[1:]])
    std_lnZ = np.r_[np.nan, std_lnZ]  # Standard deviation of only one sample does not exist.

    return {"logcummean_Z": logcummean_Z.astype(config.floatX),
            "logcumstd_Z_down": logcumstd_Z_down.astype(config.floatX),
            "logcumstd_Z_up": logcumstd_Z_up.astype(config.floatX),
            "logcumstd_Z": logstd_AIS.astype(config.floatX),
            "M_log_w_ais": M_log_w_ais,
            "lnZ_trivial": lnZ_trivial,
            "std_lnZ": std_lnZ,
            "last_sample_chain": last_sample_chain,
            "batch_size": batch_size,
            "seed": seed,
            "nb_temperatures": len(betas),
            "nb_samples": M
            }