pretrain_lr = pretrain_lr_zae layer_cost = layer_dropout.cost() trainer = GraddescentMinibatch( varin=model.varin, data=train_x, cost=layer_cost, params=layer_dropout.params_private, supervised=False, batchsize=batchsize, learningrate=pretrain_lr, momentum=momentum, rng=npy_rng ) prev_cost = numpy.inf patience = 0 for epoch in xrange(pretrain_epc): cost = trainer.epoch() if prev_cost <= cost: patience += 1 if patience > 10: patience = 0 trainer.set_learningrate(0.9 * trainer.learningrate) if trainer.learningrate < 1e-10: break prev_cost = cost save_params(model, 'ZLIN_4000_1000_4000_1000_4000_1000_4000_normhid_nolinb_cae1_dropout.npy') print "Done." ######################### # BUILD FINE-TUNE MODEL # #########################
supervised=True, cost=model.models_stack[-1].cost(), params=model.params, batchsize=batchsize, learningrate=finetune_lr, momentum=momentum, rng=npy_rng ) init_lr = trainer.learningrate prev_cost = numpy.inf for epoch in xrange(finetune_epc): cost = 0. for ipart in range(train_smp/part_size): print "part %d " % ipart, train_x.set_value(train_x_np[ipart * part_size : (ipart + 1) * part_size]) train_y.set_value(train_y_np[ipart * part_size : (ipart + 1) * part_size]) cost += trainer.epoch() print " ", cost += trainer.epoch() print "part + ", train_x.set_value(train_x_np[-part_size :]) train_y.set_value(train_y_np[-part_size :]) cost += trainer.epoch() print " ", cost += trainer.epoch() cost = cost / (train_smp/part_size + 1) / 2. if prev_cost <= cost: if trainer.learningrate < (init_lr * 1e-7): break trainer.set_learningrate(trainer.learningrate*0.8) prev_cost = cost if epoch % 10 == 0:
layer_cost = layer_dropout.cost() trainer = GraddescentMinibatch(varin=model.varin, data=train_x, cost=layer_cost, params=layer_dropout.params_private, supervised=False, batchsize=batchsize, learningrate=pretrain_lr, momentum=momentum, rng=npy_rng) prev_cost = numpy.inf patience = 0 for epoch in xrange(pretrain_epc): cost = trainer.epoch() if prev_cost <= cost: patience += 1 if patience > 10: patience = 0 trainer.set_learningrate(0.9 * trainer.learningrate) if trainer.learningrate < 1e-10: break prev_cost = cost save_params( model, 'ZLIN_4000_1000_4000_1000_4000_1000_4000_normhid_nolinb_cae1_dropout.npy' ) print "Done." #########################
varin=model.varin, data=train_x, cost=layer_cost, params=layer_dropout.params_private, supervised=False, batchsize=batchsize, learningrate=pretrain_lr, momentum=momentum, rng=npy_rng ) prev_cost = numpy.inf patience = 0 origin_x = train_x.get_value() reshaped_x = pcaback(origin_x).reshape((50000, 32, 32, 3), order='F') for epoch in xrange(pretrain_epc): cost = 0. # original data cost += trainer.epoch() # data augmentation: horizontal flip flipped_x = reshaped_x[:, ::-1, :, :].reshape((50000, 3072), order='F') train_x.set_value(pcamapping(flipped_x)) cost += trainer.epoch() # random rotation movies = numpy.zeros(reshaped_x.shape, dtype=theano.config.floatX) for i, im in enumerate(reshaped_x): angle_delta = (npy_rng.vonmises(0.0, 1.0)/(4 * numpy.pi)) * 180 movies[i] = ndimage.rotate(im, angle_delta, reshape=False, mode='wrap') train_x.set_value(pcamapping(movies.reshape((50000, 3072), order='F'))) cost += trainer.epoch() # random shift