def test_train_ae():
    ds = MNIST(which_set='train',one_hot=True,all_labelled=ALL_LABELLED,supervised=SUPERVISED)

    gsn = GSN.new(
        layer_sizes=[ds.X.shape[1], HIDDEN_SIZE,ds.X.shape[1]],
        activation_funcs=["sigmoid", "tanh", rescaled_softmax],
        pre_corruptors=[GaussianCorruptor(GAUSSIAN_NOISE)] * 3,
        post_corruptors=[SaltPepperCorruptor(SALT_PEPPER_NOISE), None,SmoothOneHotCorruptor(GAUSSIAN_NOISE)],
        layer_samplers=[BinomialSampler(), None, MultinomialSampler()],
        tied=False
    )

    _mbce = MeanBinaryCrossEntropy()
    reconstruction_cost = lambda a, b: _mbce.cost(a, b) / ds.X.shape[1]

    c = GSNCost([(0, 1.0, reconstruction_cost)], walkback=WALKBACK)

    alg = SGD(
        LEARNING_RATE,
        init_momentum=MOMENTUM,
        cost=c,
        termination_criterion=EpochCounter(MAX_EPOCHS),
        batches_per_iter=BATCHES_PER_EPOCH,
        batch_size=BATCH_SIZE,
        monitoring_dataset=ds,
        monitoring_batches=MONITORING_BATCHES
   )

    trainer = Train(ds, gsn, algorithm=alg, save_path="./results/gsn_ae_trained.pkl",
                    save_freq=5, extensions=[MonitorBasedLRAdjuster()])
    trainer.main_loop()
    print "done training"
Esempio n. 2
0
def test_train_ae():
    GC = GaussianCorruptor

    gsn = GSN.new(layer_sizes=[ds.X.shape[1], 1000],
                  activation_funcs=["sigmoid", "tanh"],
                  pre_corruptors=[None, GC(1.0)],
                  post_corruptors=[SaltPepperCorruptor(0.5),
                                   GC(1.0)],
                  layer_samplers=[BinomialSampler(), None],
                  tied=False)

    # average MBCE over example rather than sum it
    _mbce = MeanBinaryCrossEntropy()
    reconstruction_cost = lambda a, b: _mbce.cost(a, b) / ds.X.shape[1]

    c = GSNCost([(0, 1.0, reconstruction_cost)], walkback=WALKBACK)

    alg = SGD(LEARNING_RATE,
              init_momentum=MOMENTUM,
              cost=c,
              termination_criterion=EpochCounter(MAX_EPOCHS),
              batches_per_iter=BATCHES_PER_EPOCH,
              batch_size=BATCH_SIZE,
              monitoring_dataset=ds,
              monitoring_batches=10)

    trainer = Train(ds,
                    gsn,
                    algorithm=alg,
                    save_path="gsn_ae_example.pkl",
                    save_freq=5)
    trainer.main_loop()
    print "done training"
Esempio n. 3
0
def train():
    LEARNING_RATE = 1e-4
    MOMENTUM = 0.25

    MAX_EPOCHS = 500
    BATCHES_PER_EPOCH = 100
    BATCH_SIZE = 1000

    dataset = FunnelDistribution()
    cost = FunnelGSNCost([(0, 1.0, MSR())], walkback=1)

    gc = GaussianCorruptor(0.75)
    dc = DropoutCorruptor(.5)
    gsn = GSN.new([10, 200, 10],
                  [None, "tanh", "tanh"], # activation
                  [None] * 3, # pre corruption
                  [None] * 3, # post corruption
                  [None] * 3, # layer samplers
                  tied=False)
    gsn._bias_switch = False

    alg = SGD(LEARNING_RATE, init_momentum=MOMENTUM, cost=cost,
              termination_criterion=EpochCounter(MAX_EPOCHS),
              batches_per_iter=BATCHES_PER_EPOCH, batch_size=BATCH_SIZE,
              monitoring_batches=100,
              monitoring_dataset=dataset)

    trainer = Train(dataset, gsn, algorithm=alg, save_path="funnel_gsn.pkl",
                    extensions=[MonitorBasedLRAdjuster()],
                    save_freq=50)

    trainer.main_loop()
    print "done training"
Esempio n. 4
0
def test_train_ae():
    GC = GaussianCorruptor

    gsn = GSN.new(
        layer_sizes=[ds.X.shape[1], 1000],
        activation_funcs=["sigmoid", "tanh"],
        pre_corruptors=[None, GC(1.0)],
        post_corruptors=[SaltPepperCorruptor(0.5), GC(1.0)],
        layer_samplers=[BinomialSampler(), None],
        tied=False
    )

    # average MBCE over example rather than sum it
    _mbce = MeanBinaryCrossEntropy()
    reconstruction_cost = lambda a, b: _mbce.cost(a, b) / ds.X.shape[1]

    c = GSNCost([(0, 1.0, reconstruction_cost)], walkback=WALKBACK)

    alg = SGD(
        LEARNING_RATE,
        init_momentum=MOMENTUM,
        cost=c,
        termination_criterion=EpochCounter(MAX_EPOCHS),
        batches_per_iter=BATCHES_PER_EPOCH,
        batch_size=BATCH_SIZE,
        monitoring_dataset=ds,
        monitoring_batches=10
   )

    trainer = Train(ds, gsn, algorithm=alg, save_path="gsn_ae_example.pkl",
                    save_freq=5)
    trainer.main_loop()
    print "done training"
Esempio n. 5
0
def test_train_supervised():
    """
    Train a supervised GSN.
    """
    # initialize the GSN
    gsn = GSN.new(
        layer_sizes=[ds.X.shape[1], 1000, ds.y.shape[1]],
        activation_funcs=["sigmoid", "tanh", rescaled_softmax],
        pre_corruptors=[GaussianCorruptor(0.5)] * 3,
        post_corruptors=[
            SaltPepperCorruptor(.3), None,
            SmoothOneHotCorruptor(.5)
        ],
        layer_samplers=[BinomialSampler(), None,
                        MultinomialSampler()],
        tied=False)

    # average over costs rather than summing
    _rcost = MeanBinaryCrossEntropy()
    reconstruction_cost = lambda a, b: _rcost.cost(a, b) / ds.X.shape[1]

    _ccost = MeanBinaryCrossEntropy()
    classification_cost = lambda a, b: _ccost.cost(a, b) / ds.y.shape[1]

    # combine costs into GSNCost object
    c = GSNCost(
        [
            # reconstruction on layer 0 with weight 1.0
            (0, 1.0, reconstruction_cost),

            # classification on layer 2 with weight 2.0
            (2, 2.0, classification_cost)
        ],
        walkback=WALKBACK,
        mode="supervised")

    alg = SGD(
        LEARNING_RATE,
        init_momentum=MOMENTUM,
        cost=c,
        termination_criterion=EpochCounter(MAX_EPOCHS),
        batches_per_iter=BATCHES_PER_EPOCH,
        batch_size=BATCH_SIZE,
        monitoring_dataset=ds,
        monitoring_batches=10,
    )

    trainer = Train(ds,
                    gsn,
                    algorithm=alg,
                    save_path="gsn_sup_example.pkl",
                    save_freq=10,
                    extensions=[MonitorBasedLRAdjuster()])
    trainer.main_loop()
    print("done training")
Esempio n. 6
0
def test_train_supervised():
    """
    Train a supervised GSN.
    """
    # initialize the GSN
    gsn = GSN.new(
        layer_sizes=[ds.X.shape[1], 1000, ds.y.shape[1]],
        activation_funcs=["sigmoid", "tanh", rescaled_softmax],
        pre_corruptors=[GaussianCorruptor(0.5)] * 3,
        post_corruptors=[SaltPepperCorruptor(.3), None, SmoothOneHotCorruptor(.5)],
        layer_samplers=[BinomialSampler(), None, MultinomialSampler()],
        tied=False
    )

    # average over costs rather than summing
    _rcost = MeanBinaryCrossEntropy()
    reconstruction_cost = lambda a, b: _rcost.cost(a, b) / ds.X.shape[1]

    _ccost = MeanBinaryCrossEntropy()
    classification_cost = lambda a, b: _ccost.cost(a, b) / ds.y.shape[1]

    # combine costs into GSNCost object
    c = GSNCost(
        [
            # reconstruction on layer 0 with weight 1.0
            (0, 1.0, reconstruction_cost),

            # classification on layer 2 with weight 2.0
            (2, 2.0, classification_cost)
        ],
        walkback=WALKBACK,
        mode="supervised"
    )

    alg = SGD(
        LEARNING_RATE,
        init_momentum=MOMENTUM,
        cost=c,
        termination_criterion=EpochCounter(MAX_EPOCHS),
        batches_per_iter=BATCHES_PER_EPOCH,
        batch_size=BATCH_SIZE,
        monitoring_dataset=ds,
        monitoring_batches=10,
    )

    trainer = Train(ds, gsn, algorithm=alg,
                    save_path="gsn_sup_example.pkl", save_freq=10,
                    extensions=[MonitorBasedLRAdjuster()])
    trainer.main_loop()
    print "done training"