Пример #1
0
def load_init(phone, rng, whichset, wavtype, idx):
    testpath = "/u/kimtaeho/Documents/2_Project/speech_synthesis/code/datasets"
    if wavtype == "timit":
        if whichset == "noise":
            init_seq = np.random.randn(1000)
            init_seq = (init_seq - np.mean(init_seq)) / max(abs(init_seq))
            idx = "n"
        elif whichset in ["test", "train", "valid", None]:
            whichset = "test" if whichset == None else whichset
            test = np.load(os.path.join(testpath, whichset + "_wav_" + phone + ".npy"))
            test_len = test.shape[0]
            init_seq = test[idx]
            init_seq = (init_seq - _mean) / _std
    else:
        from load_data import load_data_timit_seq

        init_seq = load_data_timit_seq(stop=1, lend=1000, wavtype=wavtype)
        init_seq = init_seq.get_value().flatten()
    return init_seq, idx
Пример #2
0
def evaluate_convA():
    learning_rate = 0.0001
    n_epochs = 10000
    #dataset = 'mnist.pkl.gz'
    dataset = 'timit'
    batch_size = 10
    start = 0
    stop = batch_size 
    channel = 1
    image_h = 1
    image_w = 1000
    filter_h = 1
    filter_w = 25 
    nkerns = [300]
    corruption = {
        'Binomial' : None,
        'Gaussian' : 0.1,
        'Sequential' : None}
    wbiter = 5 
    wavtype='sin'
    learning_rule='mom'
    dechid='lin'
    postfix=''
    
    savepath = 'result/'
    savename = savepath + 'ca_test_wb'+str(wbiter)+'_toy'+wavtype+'_g'+str(corruption['Gaussian'])+'_w'+str(filter_w)+'_'+learning_rule+'_'+dechid+postfix
    if os.path.exists(savename+'.pkl'):
        ans=raw_input('Same exp. exists, continue? ([Y]/N) ')
        if ans.upper() == 'N':
            return

    nrng = np.random.RandomState(23455)
    trng = RandomStreams(nrng.randint(2 ** 30))
    if dataset == 'mnist.pkl.gz':
        from load_data import load_data_mnist
        datasets = load_data_mnist(dataset)
        train_set_x, train_set_y = datasets[0]
        valid_set_x, valid_set_y = datasets[1]
        test_set_x, test_set_y = datasets[2]
    elif dataset == 'timit':
        from load_data import load_data_timit_seq
        train_set_x = load_data_timit_seq('train', start, stop, image_w, wavtype)
        valid_set_x = load_data_timit_seq('valid', start, stop, image_w, wavtype)
        test_set_x = load_data_timit_seq('test', start, stop, image_w, wavtype)
    
    # compute number of minibatches for training, validation and testing
    n_train_batches0 = train_set_x.get_value(borrow=True).shape[0]
    n_valid_batches0 = valid_set_x.get_value(borrow=True).shape[0]
    n_test_batches0 = test_set_x.get_value(borrow=True).shape[0]
    n_train_batches = n_train_batches0 / batch_size
    n_valid_batches = n_valid_batches0 / batch_size
    n_test_batches = n_test_batches0 / batch_size

    assert min(n_train_batches, n_valid_batches, n_test_batches)>0,\
        'Maximum batch size is %d' % min(n_train_batches0, n_valid_batches0, n_test_batches0)

    # allocate symbolic variables for the data
    index = T.lscalar()  # index to a [mini]batch
    x = T.matrix('x')   # the data is presented as rasterized images

    ######################
    # BUILD ACTUAL MODEL #
    ######################
    print '... building the model'

    layer0_input = x.reshape((batch_size, channel, image_h, image_w))

    layer0 = convA(
            nrng = nrng, trng=trng,
            filter_shape = (nkerns[0], channel, filter_h, filter_w),
            poolsize = (2, 2),
            corruption = corruption)
    
    #cost = layer0.cost(layer0_input)
    cost = layer0.wbcost(layer0_input,wbiter)
    #cost = layer0.Melcost(layer0_input)
    params = layer0.params
    grads = T.grad(cost, params)
    gradsdic = dict(zip(params,grads))
    if learning_rule == 'ada':
        ad = AdaDelta()
        updates = ad.get_updates(learning_rate, gradsdic)
    elif learning_rule == 'con':
        updates = []
        for param_i, grad_i in zip(params, grads):
            updates.append((param_i, param_i - learning_rate * grad_i))
    elif learning_rule == 'mom':
        momentum = 0.96
        mm = Momentum(momentum)
        updates = mm.get_updates(learning_rate, gradsdic)

    train_model = theano.function(
        inputs = [index], 
        outputs = cost, 
        updates = updates,
        givens = {x: train_set_x[index * batch_size: (index + 1) * batch_size]})
    
    validate_model = theano.function(
        inputs = [index], 
        outputs = cost,
        givens = {x: valid_set_x[index * batch_size: (index + 1) * batch_size]})
    
    test_model = theano.function(
        inputs = [index],
        outputs = cost,
        givens = {x: test_set_x[index * batch_size: (index + 1) * batch_size]})

    ###############
    # TRAIN MODEL #
    ###############
    print '... training'
    # early-stopping parameters
    patience = 10000  # look as this many examples regardless
    patience_increase = 2  # wait this much longer when a new best is found
    improvement_threshold = 0.995  # a relative improvement of this much is
                                   # considered significant
    validation_frequency = min(n_train_batches, patience / 2)

    first_lr = learning_rate
    st_an = 800
    en_an = 2000
    best_params = None
    best_validation_loss = np.inf
    best_iter = 0
    test_score = 0.
    start_time = time.clock()
    
    score_cum=[]
    epoch = 0
    done_looping = False
    
    while (epoch < n_epochs) and (not done_looping):
        epoch = epoch + 1
        if epoch > st_an and learning_rule in ['con','mom']:
            learning_rate = first_lr/(epoch-st_an)
        #if epoch >=  st_an and epoch < en_an:
        #    learning_rate -= first_lr/(en_an-st_an)
        #elif epoch >=en_an:
        #    learning_rate = 0.
        
        for minibatch_index in xrange(n_train_batches):
            
            iter = (epoch - 1) * n_train_batches + minibatch_index
            if iter % 1000 == 0:
                print 'training @ iter = ', iter
            cost_ij = train_model(minibatch_index)

            if (iter + 1) % validation_frequency == 0:

                # compute zero-one loss on validation set
                validation_losses = [validate_model(i) for i in xrange(n_valid_batches)]
                this_validation_loss = np.mean(validation_losses)
                print('    %3i, validation error %f, %s ' % \
                      (epoch, this_validation_loss, savename))
                score_cum.append(this_validation_loss)
                plt.plot(xrange(len(score_cum)),score_cum)
                plt.savefig(savename+'.png')
                plt.close()

                # if we got the best validation score until now
                if this_validation_loss < best_validation_loss:

                    #improve patience if loss improvement is good enough
                    if this_validation_loss < best_validation_loss *  \
                       improvement_threshold:
                        patience = max(patience, iter * patience_increase)

                    # save best validation score and iteration number
                    best_validation_loss = this_validation_loss
                    best_iter = iter
                    
                    layer0.set_cost(best_validation_loss)
                    with open(savename+'.pkl', 'wb') as f:
                        pickle.dump(layer0,f)

                    # test it on the test set
                    test_losses = [test_model(i) for i in xrange(n_test_batches)]
                    test_score = np.mean(test_losses)
                    print(('       test error %f') % (test_score))

            if patience <= iter:
                done_looping = True
                break

    end_time = time.clock()
    print('Optimization complete.')
    print('Best validation score of %f obtained at iteration %i, with test performance %f' %
          (best_validation_loss, best_iter + 1, test_score))
    print >> sys.stderr, ('The code for file ' + os.path.split(__file__)[1] +
                        ' ran for %.2fm' % ((end_time - start_time) / 60.))
    print savename