示例#1
0
def testing(dataprovider_test, classifier, test_model, test_set_x, test_set_y, test_words_count, test_lines_count, batch_size, num_batches_per_bunch):
    
    print 'Testing...'
    log_likelihood = []
    test_frames_showed, progress = 0, 0
    start_test_time = time.time() # it is also stop of training time
        #for feat_lab_tuple, path in HDFDatasetDataProviderUtt(devel_files_list, valid_dataset, randomize=False, max_utt=-10):  
        #    features, labels = feat_lab_tuple 
            
    tqueue = TNetsCacheSimple.make_queue()
    cache = TNetsCacheSimple(tqueue, offset = 0, num_batches_per_bunch = 512)

    #cache.deamon = True
    cache.data_provider = dataprovider_test
    cache.start()
        
        #ex_num = 0
        
    while True:
                
	feats_lab_tuple = TNetsCacheSimple.get_elem_from_queue(tqueue)
        if isinstance(feats_lab_tuple, TNetsCacheLastElem):
           break
                    
        features, labels = feats_lab_tuple

        test_frames_showed += features.shape[0]                
        #test_set_x.set_value(features, borrow=True)
        #test_set_y.set_value(numpy.asarray(labels.flatten(), 'int32'), borrow=True)
            
        test_batches = features.shape[0] / batch_size
            #print valid_batches
            #if there is any part left in utterance (smaller than a batch_size), take it into account at the end
        if(features.shape[0] % batch_size!=0 or features.shape[0] < batch_size): 
           test_batches += 1
          
        #for i in xrange(test_batches): 
        for i in xrange(features.shape[0]):
            temp_features = []
            for j in features[i]:
                temp_features.append(classifier.projectionlayer.word_rep.get_value(borrow = True)[j])
            test_set_x.set_value(numpy.asarray(temp_features, dtype = 'float32').flatten(), borrow = True)
            test_set_y.set_value(numpy.asarray([labels[i]], dtype = 'int32'), borrow = True)
            log_likelihood.append(test_model())
        
        progress += 1
        if progress%100==0:
           end_time_test_progress = time.time()
           print 'PROGRESS: Processed %i bunches (%i frames),  TIME: %f in seconds'\
                          %(progress, test_frames_showed, end_time_test_progress - start_test_time)
        
    end_time_test_progress = time.time()
    print 'PROGRESS: Processed %i bunches (%i frames),  TIME: %f in seconds'\
                    %(progress, test_frames_showed, end_time_test_progress - start_test_time)            
    test_set_x.set_value(numpy.empty((1), 'float32'))
    test_set_y.set_value(numpy.empty((1), 'int32'))
 
    print numpy.sum(log_likelihood)
    likelihood_sum = (-numpy.sum(log_likelihood)/test_frames_showed)
    print 'likelihood_sum', likelihood_sum
示例#2
0
def training(
    dataprovider_train,
    dataprovider_valid,
    learnrate_schedular,
    classifier,
    train_model,
    validate_model,
    train_set_x,
    train_set_y,
    valid_set_x,
    valid_set_y,
    batch_size,
    num_batches_per_bunch,
    valid_words_count,
    valid_lines_count,
):

    exp_name = "fine_tuning.hdf5"
    print ".....training"
    best_valid_loss = numpy.inf
    epoch = 1
    start_time = time.time()
    while learnrate_schedular.get_rate() != 0:

        print "learning_rate:", learnrate_schedular.get_rate()
        print "epoch_number:", learnrate_schedular.epoch
        frames_showed, progress = 0, 0
        start_epoch_time = time.time()

        tqueue = TNetsCacheSimple.make_queue()
        cache = TNetsCacheSimple(
            tqueue, shuffle_frames=True, offset=0, batch_size=batch_size, num_batches_per_bunch=num_batches_per_bunch
        )
        cache.data_provider = dataprovider_train
        cache.start()

        train_cost = []
        while True:

            feats_lab_tuple = TNetsCacheSimple.get_elem_from_queue(tqueue)
            if isinstance(feats_lab_tuple, TNetsCacheLastElem):
                break

            features, labels = feats_lab_tuple
            frames_showed += features.shape[0]
            # train_batches = features.shape[0]/batch_size
            # print train_batches
            # if there is any part left in utterance (smaller than a batch_size), take it into account at the end
            # if(features.shape[0] % batch_size!=0 or features.shape[0] < batch_size):
            #    train_batches += 1
            for i in xrange(features.shape[0]):
                temp_features = []
                for j in features[i]:
                    temp_features.append(classifier.projectionlayer.word_rep.get_value(borrow=True)[j])
                train_set_x.set_value(numpy.asarray(temp_features, dtype="float32").flatten(), borrow=True)
                train_set_y.set_value(numpy.asarray([labels[i]], dtype="int32"), borrow=True)
                train_model(learnrate_schedular.get_rate())

            progress += 1
            if progress % 500 == 0:
                end_time_progress = time.time()
                print "PROGRESS: Processed %i bunches (%i frames), TIME: %f in seconds" % (
                    progress,
                    frames_showed,
                    (end_time_progress - start_epoch_time),
                )

        end_time_progress = time.time()
        print "PROGRESS: Processed %i bunches (%i frames), TIME: %f in seconds" % (
            progress,
            frames_showed,
            (end_time_progress - start_epoch_time),
        )
        train_set_x.set_value(numpy.empty((1), dtype="float32"))
        train_set_y.set_value(numpy.empty((1), dtype="int32"))
        classifier_name = "MLP" + str(learnrate_schedular.epoch)
        work_dir = "/afs/inf.ed.ac.uk/user/s12/s1264845/scratch/s1264845/mlp6/nnets"
        save_mlp(classifier, work_dir + exp_name, classifier_name)

        print "Validating..."
        valid_losses = []
        log_likelihood = []
        valid_frames_showed, progress = 0, 0
        start_valid_time = time.time()  # it is also stop of training time

        tqueue = TNetsCacheSimple.make_queue()
        cache = TNetsCacheSimple(tqueue, offset=0, num_batches_per_bunch=512)

        cache.data_provider = dataprovider_valid
        cache.start()

        while True:

            feats_lab_tuple = TNetsCacheSimple.get_elem_from_queue(tqueue)
            if isinstance(feats_lab_tuple, TNetsCacheLastElem):
                break

            features, labels = feats_lab_tuple

            valid_frames_showed += features.shape[0]
            # valid_batches = features.shape[0] / batch_size
            # print valid_batches
            # if there is any part left in utterance (smaller than a batch_size), take it into account at the end
            # if(features.shape[0] % batch_size!=0 or features.shape[0] < batch_size):
            #    valid_batches += 1

            for i in xrange(features.shape[0]):
                temp_features = []
                for j in features[i]:
                    temp_features.append(classifier.projectionlayer.word_rep.get_value(borrow=True)[j])
                valid_set_x.set_value(numpy.asarray(temp_features, dtype="float32").flatten(), borrow=True)
                valid_set_y.set_value(numpy.asarray([labels[i]], dtype="int32"), borrow=True)
                out = validate_model()
                error_rate = out[0]
                likelihoods = out[1]
                valid_losses.append(error_rate)
                log_likelihood.append(likelihoods)
                # save_posteriors(likelihoods, GlobalCfg.get_working_dir() + posterior_path, str(ex_num), str(learnrate_schedular.epoch))

            progress += 1
            if progress % 100 == 0:
                end_time_valid_progress = time.time()
                print "PROGRESS: Processed %i bunches (%i frames),  TIME: %f in seconds" % (
                    progress,
                    valid_frames_showed,
                    end_time_valid_progress - start_valid_time,
                )

        end_time_valid_progress = time.time()
        print "PROGRESS: Processed %i bunches (%i frames),  TIME: %f in seconds" % (
            progress,
            valid_frames_showed,
            end_time_valid_progress - start_valid_time,
        )
        valid_set_x.set_value(numpy.empty((1), "float32"))
        valid_set_y.set_value(numpy.empty((1), "int32"))

        end_epoch_time = time.time()
        print "time taken for this epoch in seconds: %f" % (end_epoch_time - start_epoch_time)

        this_validation_loss = numpy.mean(valid_losses)
        entropy = -numpy.sum(log_likelihood) / valid_frames_showed
        print this_validation_loss, entropy, numpy.sum(log_likelihood)
        # loglikelihood_sum = numpy.sum(log_likelihood)
        # print 'error_rate:', this_validation_loss

        if entropy < best_valid_loss:
            # learning_rate = learnrate_schedular.get_next_rate(this_validation_loss * 100.)
            learning_rate = learnrate_schedular.get_next_rate(entropy)
            best_valid_loss = entropy
            # best_epoch = learnrate_schedular.epoch-1
        else:
            # learnrate_schedular.epoch = learnrate_schedular.epoch + 1
            learnrate_schedular.rate = 0.0
    end_time = time.time()
    print "The fine tuning ran for %.2fm" % ((end_time - start_time) / 60.0)
示例#3
0
def training(dataprovider_train, dataprovider_valid, learnrate_schedular,
             classifier, train_model, validate_model, train_set_x, train_set_y,
             valid_set_x, valid_set_y, batch_size, num_batches_per_bunch,
             valid_words_count, valid_lines_count):

    exp_name = 'fine_tuning.hdf5'
    print '.....training'
    best_valid_loss = numpy.inf
    epoch = 1
    start_time = time.time()
    while (learnrate_schedular.get_rate() != 0):

        print 'learning_rate:', learnrate_schedular.get_rate()
        print 'epoch_number:', learnrate_schedular.epoch
        frames_showed, progress = 0, 0
        start_epoch_time = time.time()

        tqueue = TNetsCacheSimple.make_queue()
        cache = TNetsCacheSimple(tqueue, shuffle_frames = True, offset=0, \
                                 batch_size = batch_size, num_batches_per_bunch = num_batches_per_bunch)
        cache.data_provider = dataprovider_train
        cache.start()

        train_cost = []
        while True:

            feats_lab_tuple = TNetsCacheSimple.get_elem_from_queue(tqueue)
            if isinstance(feats_lab_tuple, TNetsCacheLastElem):
                break

            features, labels = feats_lab_tuple
            frames_showed += features.shape[0]
            #train_batches = features.shape[0]/batch_size
            #print train_batches
            #if there is any part left in utterance (smaller than a batch_size), take it into account at the end
            #if(features.shape[0] % batch_size!=0 or features.shape[0] < batch_size):
            #    train_batches += 1
            for i in xrange(features.shape[0]):
                temp_features = []
                for j in features[i]:
                    temp_features.append(
                        classifier.projectionlayer.word_rep.get_value(
                            borrow=True)[j])
                train_set_x.set_value(numpy.asarray(temp_features,
                                                    dtype='float32').flatten(),
                                      borrow=True)
                train_set_y.set_value(numpy.asarray([labels[i]],
                                                    dtype='int32'),
                                      borrow=True)
                train_model(learnrate_schedular.get_rate())

            progress += 1
            if progress % 500 == 0:
                end_time_progress = time.time()
                print 'PROGRESS: Processed %i bunches (%i frames), TIME: %f in seconds'\
                          %(progress, frames_showed,(end_time_progress-start_epoch_time))

        end_time_progress = time.time()
        print 'PROGRESS: Processed %i bunches (%i frames), TIME: %f in seconds'\
                          %(progress, frames_showed,(end_time_progress-start_epoch_time))
        train_set_x.set_value(numpy.empty((1), dtype='float32'))
        train_set_y.set_value(numpy.empty((1), dtype='int32'))
        classifier_name = 'MLP' + str(learnrate_schedular.epoch)
        work_dir = '/afs/inf.ed.ac.uk/user/s12/s1264845/scratch/s1264845/mlp6/nnets'
        save_mlp(classifier, work_dir + exp_name, classifier_name)

        print 'Validating...'
        valid_losses = []
        log_likelihood = []
        valid_frames_showed, progress = 0, 0
        start_valid_time = time.time()  # it is also stop of training time

        tqueue = TNetsCacheSimple.make_queue()
        cache = TNetsCacheSimple(tqueue, offset=0, num_batches_per_bunch=512)

        cache.data_provider = dataprovider_valid
        cache.start()

        while True:

            feats_lab_tuple = TNetsCacheSimple.get_elem_from_queue(tqueue)
            if isinstance(feats_lab_tuple, TNetsCacheLastElem):
                break

            features, labels = feats_lab_tuple

            valid_frames_showed += features.shape[0]
            #valid_batches = features.shape[0] / batch_size
            #print valid_batches
            #if there is any part left in utterance (smaller than a batch_size), take it into account at the end
            #if(features.shape[0] % batch_size!=0 or features.shape[0] < batch_size):
            #    valid_batches += 1

            for i in xrange(features.shape[0]):
                temp_features = []
                for j in features[i]:
                    temp_features.append(
                        classifier.projectionlayer.word_rep.get_value(
                            borrow=True)[j])
                valid_set_x.set_value(numpy.asarray(temp_features,
                                                    dtype='float32').flatten(),
                                      borrow=True)
                valid_set_y.set_value(numpy.asarray([labels[i]],
                                                    dtype='int32'),
                                      borrow=True)
                out = validate_model()
                error_rate = out[0]
                likelihoods = out[1]
                valid_losses.append(error_rate)
                log_likelihood.append(likelihoods)
                #save_posteriors(likelihoods, GlobalCfg.get_working_dir() + posterior_path, str(ex_num), str(learnrate_schedular.epoch))

            progress += 1
            if progress % 100 == 0:
                end_time_valid_progress = time.time()
                print 'PROGRESS: Processed %i bunches (%i frames),  TIME: %f in seconds'\
                          %(progress, valid_frames_showed, end_time_valid_progress - start_valid_time)

        end_time_valid_progress = time.time()
        print 'PROGRESS: Processed %i bunches (%i frames),  TIME: %f in seconds'\
                          %(progress, valid_frames_showed, end_time_valid_progress - start_valid_time)
        valid_set_x.set_value(numpy.empty((1), 'float32'))
        valid_set_y.set_value(numpy.empty((1), 'int32'))

        end_epoch_time = time.time()
        print 'time taken for this epoch in seconds: %f' % (end_epoch_time -
                                                            start_epoch_time)

        this_validation_loss = numpy.mean(valid_losses)
        entropy = (-numpy.sum(log_likelihood) / valid_frames_showed)
        print this_validation_loss, entropy, numpy.sum(log_likelihood)
        #loglikelihood_sum = numpy.sum(log_likelihood)
        #print 'error_rate:', this_validation_loss

        if entropy < best_valid_loss:
            #learning_rate = learnrate_schedular.get_next_rate(this_validation_loss * 100.)
            learning_rate = learnrate_schedular.get_next_rate(entropy)
            best_valid_loss = entropy
        #best_epoch = learnrate_schedular.epoch-1
        else:
            #learnrate_schedular.epoch = learnrate_schedular.epoch + 1
            learnrate_schedular.rate = 0.0
    end_time = time.time()
    print 'The fine tuning ran for %.2fm' % ((end_time - start_time) / 60.)
示例#4
0
def train_mlp(L1_reg = 0.0, L2_reg = 0.0000, num_batches_per_bunch = 512, batch_size = 1, num_bunches_queue = 5, offset = 0, path_name = '/afs/inf.ed.ac.uk/user/s12/s1264845/scratch/s1264845/data/'):
    

    voc_list = Vocabulary(path_name + 'train')
    voc_list.vocab_create()
    vocab = voc_list.vocab
    vocab_size = voc_list.vocab_size
    
    voc_list_valid = Vocabulary(path_name + 'valid')
    voc_list_valid.vocab_create()
    count = voc_list_valid.count

    voc_list_test = Vocabulary(path_name + 'test')
    voc_list_test.vocab_create()
    no_test_tokens = voc_list_test.count
    print 'The number of sentenses in test set:', no_test_tokens
 
    #print 'number of words in valid data:', count 
    dataprovider_train = DataProvider(path_name + 'train', vocab, vocab_size )
    dataprovider_valid = DataProvider(path_name + 'valid', vocab, vocab_size )
    dataprovider_test = DataProvider(path_name + 'test', vocab, vocab_size )

    #learn_list = [0.1, 0.1, 0.1, 0.75, 0.5, 0.25, 0.125, 0.0625, 0]
    exp_name = 'fine_tuning.hdf5'
    posterior_path = 'log_likelihoods'
    print '..building the model'

    #symbolic variables for input, target vector and batch index
    index = T.lscalar('index')
    x = T.fmatrix('x')
    y = T.ivector('y')
    learning_rate = T.fscalar('learning_rate') 

    #theano shares variables for train, valid and test
    train_set_x = theano.shared(numpy.empty((1,1), dtype='float32'), allow_downcast = True)
    train_set_y = theano.shared(numpy.empty((1), dtype = 'int32'), allow_downcast = True)
    
    valid_set_x = theano.shared(numpy.empty((1,1), dtype='float32'), allow_downcast = True)
    valid_set_y = theano.shared(numpy.empty((1), dtype = 'int32'), allow_downcast = True)
    
    test_set_x = theano.shared(numpy.empty((1,1), dtype='float32'), allow_downcast = True)
    test_set_y = theano.shared(numpy.empty((1), dtype = 'int32'), allow_downcast = True)
    
    rng = numpy.random.RandomState(1234) 
   
    classifier = MLP(rng = rng, input = x, n_in = vocab_size, n_hidden1 = 30, n_hidden2 = 60 , n_out = vocab_size)
    #classifier = MLP(rng = rng, input = x, n_in = vocab_size, n_hidden = 60, n_out = vocab_size)

    cost = classifier.negative_log_likelihood(y) + L1_reg * classifier.L1 + L2_reg * classifier.L2_sqr
    
    #constructor for learning rate class
    learnrate_schedular = LearningRateNewBob(start_rate=0.001, scale_by=.5, max_epochs=9999,\
                                    min_derror_ramp_start=.1, min_derror_stop=.1, init_error=100.)

    #learnrate_schedular = LearningRateList(learn_list)

    frame_error = classifier.errors(y)
    likelihood = classifier.sum(y)

    #test model
    test_model = theano.function(inputs = [index], outputs = likelihood,  \
                                 givens = {x: test_set_x[index * batch_size:(index + 1) * batch_size],
                                           y: test_set_y[index * batch_size:(index + 1) * batch_size]})
    #validation_model
    validate_model = theano.function(inputs = [index], outputs = [frame_error, likelihood], \
                                     givens = {x: valid_set_x[index * batch_size:(index + 1) * batch_size],
                                               y: valid_set_y[index * batch_size:(index + 1) * batch_size]})

    gradient_param = []
    #calculates the gradient of cost with respect to parameters 
    for param in classifier.params:
        gradient_param.append(T.cast(T.grad(cost, param), 'float32'))
        
    updates = []
    
    for param, gradient in zip(classifier.params, gradient_param):
        updates.append((param, param - learning_rate * gradient))
    
    #training_model
    train_model = theano.function(inputs = [index, theano.Param(learning_rate, default = 0.01)], outputs = cost, updates = updates, \
                                 givens = {x: train_set_x[index * batch_size:(index + 1) * batch_size],
                                           y: train_set_y[index * batch_size:(index + 1) * batch_size]})
   

    #theano.printing.pydotprint(train_model, outfile = "pics/train.png", var_with_name_simple = True) 
    #path_save = '/afs/inf.ed.ac.uk/user/s12/s1264845/scratch/s1264845/mlp/saved_weights/' 
    print '.....training'    
    best_valid_loss = numpy.inf    
    epoch = 1
    start_time = time.time()
    while(learnrate_schedular.get_rate() != 0):
	
	print 'learning_rate:', learnrate_schedular.get_rate()
        print 'epoch_number:', learnrate_schedular.epoch
        
        frames_showed, progress = 0, 0
        start_epoch_time = time.time()
        
        tqueue = TNetsCacheSimple.make_queue()
        cache = TNetsCacheSimple(tqueue, shuffle_frames = True, offset=0, \
                                 batch_size = batch_size, num_batches_per_bunch = num_batches_per_bunch) 
        cache.data_provider = dataprovider_train
        cache.start()
        
        train_cost = []
        while True:

            feats_lab_tuple = TNetsCacheSimple.get_elem_from_queue(tqueue)
            if isinstance(feats_lab_tuple, TNetsCacheLastElem):
                break
                
            features, labels = feats_lab_tuple                  
            train_set_x.set_value(features, borrow=True)
            train_set_y.set_value(numpy.asarray(labels.flatten(), dtype = 'int32'), borrow=True)
            
            frames_showed += features.shape[0]
            train_batches = features.shape[0]/batch_size
            #print train_batches
                #if there is any part left in utterance (smaller than a batch_size), take it into account at the end
            if(features.shape[0] % batch_size!=0 or features.shape[0] < batch_size): 
                train_batches += 1
            
            for i in xrange(train_batches):
                #train_cost.append(train_model(i, learnrate_schedular.get_rate()))
                train_model(i, learnrate_schedular.get_rate())               
            progress += 1
            if progress%10==0:
                end_time_progress = time.time()
                print 'PROGRESS: Processed %i bunches (%i frames), TIME: %f in seconds'\
                          %(progress, frames_showed,(end_time_progress-start_epoch_time))
        
        end_time_progress = time.time()
	print 'PROGRESS: Processed %i bunches (%i frames), TIME: %f in seconds'\
                          %(progress, frames_showed,(end_time_progress-start_epoch_time))
        train_set_x.set_value(numpy.empty((1,1), dtype = 'float32'))
        train_set_y.set_value(numpy.empty((1), dtype = 'int32'))
	classifier_name = 'MLP' + str(learnrate_schedular.epoch)
	
	save_mlp(classifier, GlobalCfg.get_working_dir()+exp_name , classifier_name)
                  
        print 'Validating...'
        valid_losses = []
	log_likelihood = []
        valid_frames_showed, progress = 0, 0
        start_valid_time = time.time() # it is also stop of training time
        #for feat_lab_tuple, path in HDFDatasetDataProviderUtt(devel_files_list, valid_dataset, randomize=False, max_utt=-10):  
        #    features, labels = feat_lab_tuple 
            
        tqueue = TNetsCacheSimple.make_queue()
        cache = TNetsCacheSimple(tqueue, offset = 0, num_batches_per_bunch = 16)

        #cache.deamon = True
        cache.data_provider = dataprovider_valid
        cache.start()
        
        #ex_num = 0
        
        while True:
                
            feats_lab_tuple = TNetsCacheSimple.get_elem_from_queue(tqueue)
            if isinstance(feats_lab_tuple, TNetsCacheLastElem):
                break
                    
            features, labels = feats_lab_tuple

            valid_frames_showed += features.shape[0]                
            valid_set_x.set_value(features, borrow=True)
            valid_set_y.set_value(numpy.asarray(labels.flatten(), 'int32'), borrow=True)
            
            valid_batches = features.shape[0] / batch_size
            #print valid_batches
            #if there is any part left in utterance (smaller than a batch_size), take it into account at the end
            if(features.shape[0] % batch_size!=0 or features.shape[0] < batch_size): 
                valid_batches += 1
          
            for i in xrange(valid_batches):
                #ex_num = ex_num + 1
                out = validate_model(i)
		error_rate = out[0]
		likelihoods = out[1] 
                valid_losses.append(error_rate)
		log_likelihood.append(likelihoods)
                #save_posteriors(likelihoods, GlobalCfg.get_working_dir() + posterior_path, str(ex_num), str(learnrate_schedular.epoch))
                
	    
            progress += 1
            if progress%10==0:
                end_time_valid_progress = time.time()
                print 'PROGRESS: Processed %i bunches (%i frames),  TIME: %f in seconds'\
                          %(progress, valid_frames_showed, end_time_valid_progress - start_valid_time)
        
        end_time_valid_progress = time.time()
        print 'PROGRESS: Processed %i bunches (%i frames),  TIME: %f in seconds'\
                          %(progress, valid_frames_showed, end_time_valid_progress - start_valid_time)            
        valid_set_x.set_value(numpy.empty((1,1), 'float32'))
        valid_set_y.set_value(numpy.empty((1), 'int32'))
            
            
        end_epoch_time = time.time()
	print 'time taken for this epoch in seconds: %f' %(end_epoch_time - start_epoch_time)
            
        this_validation_loss = numpy.mean(valid_losses)
	loglikelihood_sum = numpy.sum(log_likelihood)
	#ppl = math.exp(- loglikelihood_sum /count)
	#print 'ppl:', ppl
	print 'error_rate:', this_validation_loss
	print 'valid log likelihood:', loglikelihood_sum     
	#print 'mean log_probability', this_validation_loss 
        #learnrate_schedular.get_next_rate(this_validation_loss * 100.)
	    #learnrate_schedular.get_next_rate()
	    #print 'epoch_number:', learnrate_schedular.epoch
                
            # logger.info('Epoch %i (lr: %f) took %f min (SPEED [presentations/second] training %f, cv %f), cv error %f %%' % \
            #         (self.cfg.finetune_scheduler.epoch-1, self.cfg.finetune_scheduler.get_rate(), \
            #          ((end_epoch_time-start_epoch_time)/60.0), (frames_showed/(start_valid_time-start_epoch_time)), \
            #          (valid_frames_showed/(stop_valid_time-start_valid_time)), this_validation_loss*100.))

            #self.cfg.finetune_scheduler.get_next_rate(this_validation_loss*100.)
        if this_validation_loss < best_valid_loss:
	   learning_rate = learnrate_schedular.get_next_rate(this_validation_loss * 100.)
           best_valid_loss = this_validation_loss
           #best_epoch = learnrate_schedular.epoch-1
	else:
           #learnrate_schedular.epoch = learnrate_schedular.epoch + 1
	   learnrate_schedular.rate = 0.0
    
    end_time = time.time()
        
    #print 'Optimization complete with best validation score of %f %%' %  best_valid_loss * 100.
    print 'The fine tuning ran for %.2fm' %((end_time-start_time)/60.)

    print 'Testing...'
    log_likelihood_test = []
    test_frames_showed, progress = 0, 0
    start_test_time = time.time() # it is also stop of training time
        #for feat_lab_tuple, path in HDFDatasetDataProviderUtt(devel_files_list, valid_dataset, randomize=False, max_utt=-10):  
        #    features, labels = feat_lab_tuple 
            
    tqueue = TNetsCacheSimple.make_queue()
    cache = TNetsCacheSimple(tqueue, offset = 0, num_batches_per_bunch = 16)

    #cache.deamon = True
    cache.data_provider = dataprovider_test
    cache.start()
        
        #ex_num = 0
        
    while True:
                
	feats_lab_tuple = TNetsCacheSimple.get_elem_from_queue(tqueue)
        if isinstance(feats_lab_tuple, TNetsCacheLastElem):
           break
                    
        features, labels = feats_lab_tuple

        test_frames_showed += features.shape[0]                
        test_set_x.set_value(features, borrow=True)
        test_set_y.set_value(numpy.asarray(labels.flatten(), 'int32'), borrow=True)
            
        test_batches = features.shape[0] / batch_size
            #print valid_batches
            #if there is any part left in utterance (smaller than a batch_size), take it into account at the end
        if(features.shape[0] % batch_size!=0 or features.shape[0] < batch_size): 
           test_batches += 1
          
        for i in xrange(test_batches): 
            log_likelihood_test.append(test_model(i))
	    
        progress += 1
        if progress%10==0:
           end_time_test_progress = time.time()
           print 'PROGRESS: Processed %i bunches (%i frames),  TIME: %f in seconds'\
                          %(progress, test_frames_showed, end_time_test_progress - start_test_time)
        
    end_time_test_progress = time.time()
    print 'PROGRESS: Processed %i bunches (%i frames),  TIME: %f in seconds'\
                    %(progress, test_frames_showed, end_time_test_progress - start_test_time)            
    test_set_x.set_value(numpy.empty((1,1), 'float32'))
    test_set_y.set_value(numpy.empty((1), 'int32'))
 
    likelihood_sum = numpy.sum(log_likelihood_test)
    print 'likelihood_sum', likelihood_sum
示例#5
0
def train_mlp(
        L1_reg=0.0,
        L2_reg=0.0000,
        num_batches_per_bunch=512,
        batch_size=1,
        num_bunches_queue=5,
        offset=0,
        path_name='/afs/inf.ed.ac.uk/user/s12/s1264845/scratch/s1264845/data/'
):

    voc_list = Vocabulary(path_name + 'train')
    voc_list.vocab_create()
    vocab = voc_list.vocab
    vocab_size = voc_list.vocab_size

    voc_list_valid = Vocabulary(path_name + 'valid')
    voc_list_valid.vocab_create()
    count = voc_list_valid.count

    voc_list_test = Vocabulary(path_name + 'test')
    voc_list_test.vocab_create()
    no_test_tokens = voc_list_test.count
    print 'The number of sentenses in test set:', no_test_tokens

    #print 'number of words in valid data:', count
    dataprovider_train = DataProvider(path_name + 'train', vocab, vocab_size)
    dataprovider_valid = DataProvider(path_name + 'valid', vocab, vocab_size)
    dataprovider_test = DataProvider(path_name + 'test', vocab, vocab_size)

    #learn_list = [0.1, 0.1, 0.1, 0.75, 0.5, 0.25, 0.125, 0.0625, 0]
    exp_name = 'fine_tuning.hdf5'
    posterior_path = 'log_likelihoods'
    print '..building the model'

    #symbolic variables for input, target vector and batch index
    index = T.lscalar('index')
    x = T.fmatrix('x')
    y = T.ivector('y')
    learning_rate = T.fscalar('learning_rate')

    #theano shares variables for train, valid and test
    train_set_x = theano.shared(numpy.empty((1, 1), dtype='float32'),
                                allow_downcast=True)
    train_set_y = theano.shared(numpy.empty((1), dtype='int32'),
                                allow_downcast=True)

    valid_set_x = theano.shared(numpy.empty((1, 1), dtype='float32'),
                                allow_downcast=True)
    valid_set_y = theano.shared(numpy.empty((1), dtype='int32'),
                                allow_downcast=True)

    test_set_x = theano.shared(numpy.empty((1, 1), dtype='float32'),
                               allow_downcast=True)
    test_set_y = theano.shared(numpy.empty((1), dtype='int32'),
                               allow_downcast=True)

    rng = numpy.random.RandomState(1234)

    classifier = MLP(rng=rng,
                     input=x,
                     n_in=vocab_size,
                     n_hidden1=30,
                     n_hidden2=60,
                     n_out=vocab_size)
    #classifier = MLP(rng = rng, input = x, n_in = vocab_size, n_hidden = 60, n_out = vocab_size)

    cost = classifier.negative_log_likelihood(
        y) + L1_reg * classifier.L1 + L2_reg * classifier.L2_sqr

    #constructor for learning rate class
    learnrate_schedular = LearningRateNewBob(start_rate=0.001, scale_by=.5, max_epochs=9999,\
                                    min_derror_ramp_start=.1, min_derror_stop=.1, init_error=100.)

    #learnrate_schedular = LearningRateList(learn_list)

    frame_error = classifier.errors(y)
    likelihood = classifier.sum(y)

    #test model
    test_model = theano.function(inputs = [index], outputs = likelihood,  \
                                 givens = {x: test_set_x[index * batch_size:(index + 1) * batch_size],
                                           y: test_set_y[index * batch_size:(index + 1) * batch_size]})
    #validation_model
    validate_model = theano.function(inputs = [index], outputs = [frame_error, likelihood], \
                                     givens = {x: valid_set_x[index * batch_size:(index + 1) * batch_size],
                                               y: valid_set_y[index * batch_size:(index + 1) * batch_size]})

    gradient_param = []
    #calculates the gradient of cost with respect to parameters
    for param in classifier.params:
        gradient_param.append(T.cast(T.grad(cost, param), 'float32'))

    updates = []

    for param, gradient in zip(classifier.params, gradient_param):
        updates.append((param, param - learning_rate * gradient))

    #training_model
    train_model = theano.function(inputs = [index, theano.Param(learning_rate, default = 0.01)], outputs = cost, updates = updates, \
                                 givens = {x: train_set_x[index * batch_size:(index + 1) * batch_size],
                                           y: train_set_y[index * batch_size:(index + 1) * batch_size]})

    #theano.printing.pydotprint(train_model, outfile = "pics/train.png", var_with_name_simple = True)
    #path_save = '/afs/inf.ed.ac.uk/user/s12/s1264845/scratch/s1264845/mlp/saved_weights/'
    print '.....training'
    best_valid_loss = numpy.inf
    epoch = 1
    start_time = time.time()
    while (learnrate_schedular.get_rate() != 0):

        print 'learning_rate:', learnrate_schedular.get_rate()
        print 'epoch_number:', learnrate_schedular.epoch

        frames_showed, progress = 0, 0
        start_epoch_time = time.time()

        tqueue = TNetsCacheSimple.make_queue()
        cache = TNetsCacheSimple(tqueue, shuffle_frames = True, offset=0, \
                                 batch_size = batch_size, num_batches_per_bunch = num_batches_per_bunch)
        cache.data_provider = dataprovider_train
        cache.start()

        train_cost = []
        while True:

            feats_lab_tuple = TNetsCacheSimple.get_elem_from_queue(tqueue)
            if isinstance(feats_lab_tuple, TNetsCacheLastElem):
                break

            features, labels = feats_lab_tuple
            train_set_x.set_value(features, borrow=True)
            train_set_y.set_value(numpy.asarray(labels.flatten(),
                                                dtype='int32'),
                                  borrow=True)

            frames_showed += features.shape[0]
            train_batches = features.shape[0] / batch_size
            #print train_batches
            #if there is any part left in utterance (smaller than a batch_size), take it into account at the end
            if (features.shape[0] % batch_size != 0
                    or features.shape[0] < batch_size):
                train_batches += 1

            for i in xrange(train_batches):
                #train_cost.append(train_model(i, learnrate_schedular.get_rate()))
                train_model(i, learnrate_schedular.get_rate())
            progress += 1
            if progress % 10 == 0:
                end_time_progress = time.time()
                print 'PROGRESS: Processed %i bunches (%i frames), TIME: %f in seconds'\
                          %(progress, frames_showed,(end_time_progress-start_epoch_time))

        end_time_progress = time.time()
        print 'PROGRESS: Processed %i bunches (%i frames), TIME: %f in seconds'\
                                 %(progress, frames_showed,(end_time_progress-start_epoch_time))
        train_set_x.set_value(numpy.empty((1, 1), dtype='float32'))
        train_set_y.set_value(numpy.empty((1), dtype='int32'))
        classifier_name = 'MLP' + str(learnrate_schedular.epoch)

        save_mlp(classifier,
                 GlobalCfg.get_working_dir() + exp_name, classifier_name)

        print 'Validating...'
        valid_losses = []
        log_likelihood = []
        valid_frames_showed, progress = 0, 0
        start_valid_time = time.time()  # it is also stop of training time
        #for feat_lab_tuple, path in HDFDatasetDataProviderUtt(devel_files_list, valid_dataset, randomize=False, max_utt=-10):
        #    features, labels = feat_lab_tuple

        tqueue = TNetsCacheSimple.make_queue()
        cache = TNetsCacheSimple(tqueue, offset=0, num_batches_per_bunch=16)

        #cache.deamon = True
        cache.data_provider = dataprovider_valid
        cache.start()

        #ex_num = 0

        while True:

            feats_lab_tuple = TNetsCacheSimple.get_elem_from_queue(tqueue)
            if isinstance(feats_lab_tuple, TNetsCacheLastElem):
                break

            features, labels = feats_lab_tuple

            valid_frames_showed += features.shape[0]
            valid_set_x.set_value(features, borrow=True)
            valid_set_y.set_value(numpy.asarray(labels.flatten(), 'int32'),
                                  borrow=True)

            valid_batches = features.shape[0] / batch_size
            #print valid_batches
            #if there is any part left in utterance (smaller than a batch_size), take it into account at the end
            if (features.shape[0] % batch_size != 0
                    or features.shape[0] < batch_size):
                valid_batches += 1

            for i in xrange(valid_batches):
                #ex_num = ex_num + 1
                out = validate_model(i)
                error_rate = out[0]
                likelihoods = out[1]
                valid_losses.append(error_rate)
                log_likelihood.append(likelihoods)
                #save_posteriors(likelihoods, GlobalCfg.get_working_dir() + posterior_path, str(ex_num), str(learnrate_schedular.epoch))

            progress += 1
            if progress % 10 == 0:
                end_time_valid_progress = time.time()
                print 'PROGRESS: Processed %i bunches (%i frames),  TIME: %f in seconds'\
                          %(progress, valid_frames_showed, end_time_valid_progress - start_valid_time)

        end_time_valid_progress = time.time()
        print 'PROGRESS: Processed %i bunches (%i frames),  TIME: %f in seconds'\
                          %(progress, valid_frames_showed, end_time_valid_progress - start_valid_time)
        valid_set_x.set_value(numpy.empty((1, 1), 'float32'))
        valid_set_y.set_value(numpy.empty((1), 'int32'))

        end_epoch_time = time.time()
        print 'time taken for this epoch in seconds: %f' % (end_epoch_time -
                                                            start_epoch_time)

        this_validation_loss = numpy.mean(valid_losses)
        loglikelihood_sum = numpy.sum(log_likelihood)
        #ppl = math.exp(- loglikelihood_sum /count)
        #print 'ppl:', ppl
        print 'error_rate:', this_validation_loss
        print 'valid log likelihood:', loglikelihood_sum
        #print 'mean log_probability', this_validation_loss
        #learnrate_schedular.get_next_rate(this_validation_loss * 100.)
        #learnrate_schedular.get_next_rate()
        #print 'epoch_number:', learnrate_schedular.epoch

        # logger.info('Epoch %i (lr: %f) took %f min (SPEED [presentations/second] training %f, cv %f), cv error %f %%' % \
        #         (self.cfg.finetune_scheduler.epoch-1, self.cfg.finetune_scheduler.get_rate(), \
        #          ((end_epoch_time-start_epoch_time)/60.0), (frames_showed/(start_valid_time-start_epoch_time)), \
        #          (valid_frames_showed/(stop_valid_time-start_valid_time)), this_validation_loss*100.))

        #self.cfg.finetune_scheduler.get_next_rate(this_validation_loss*100.)
        if this_validation_loss < best_valid_loss:
            learning_rate = learnrate_schedular.get_next_rate(
                this_validation_loss * 100.)
            best_valid_loss = this_validation_loss
    #best_epoch = learnrate_schedular.epoch-1
        else:
            #learnrate_schedular.epoch = learnrate_schedular.epoch + 1
            learnrate_schedular.rate = 0.0

    end_time = time.time()

    #print 'Optimization complete with best validation score of %f %%' %  best_valid_loss * 100.
    print 'The fine tuning ran for %.2fm' % ((end_time - start_time) / 60.)

    print 'Testing...'
    log_likelihood_test = []
    test_frames_showed, progress = 0, 0
    start_test_time = time.time()  # it is also stop of training time
    #for feat_lab_tuple, path in HDFDatasetDataProviderUtt(devel_files_list, valid_dataset, randomize=False, max_utt=-10):
    #    features, labels = feat_lab_tuple

    tqueue = TNetsCacheSimple.make_queue()
    cache = TNetsCacheSimple(tqueue, offset=0, num_batches_per_bunch=16)

    #cache.deamon = True
    cache.data_provider = dataprovider_test
    cache.start()

    #ex_num = 0

    while True:

        feats_lab_tuple = TNetsCacheSimple.get_elem_from_queue(tqueue)
        if isinstance(feats_lab_tuple, TNetsCacheLastElem):
            break

        features, labels = feats_lab_tuple

        test_frames_showed += features.shape[0]
        test_set_x.set_value(features, borrow=True)
        test_set_y.set_value(numpy.asarray(labels.flatten(), 'int32'),
                             borrow=True)

        test_batches = features.shape[0] / batch_size
        #print valid_batches
        #if there is any part left in utterance (smaller than a batch_size), take it into account at the end
        if (features.shape[0] % batch_size != 0
                or features.shape[0] < batch_size):
            test_batches += 1

        for i in xrange(test_batches):
            log_likelihood_test.append(test_model(i))

        progress += 1
        if progress % 10 == 0:
            end_time_test_progress = time.time()
            print 'PROGRESS: Processed %i bunches (%i frames),  TIME: %f in seconds'\
                           %(progress, test_frames_showed, end_time_test_progress - start_test_time)

    end_time_test_progress = time.time()
    print 'PROGRESS: Processed %i bunches (%i frames),  TIME: %f in seconds'\
                    %(progress, test_frames_showed, end_time_test_progress - start_test_time)
    test_set_x.set_value(numpy.empty((1, 1), 'float32'))
    test_set_y.set_value(numpy.empty((1), 'int32'))

    likelihood_sum = numpy.sum(log_likelihood_test)
    print 'likelihood_sum', likelihood_sum
示例#6
0
def testing(dataprovider_test, classifier, test_model, test_set_x, test_set_y,
            test_words_count, test_lines_count, batch_size,
            num_batches_per_bunch):

    print 'Testing...'
    log_likelihood = []
    test_frames_showed, progress = 0, 0
    start_test_time = time.time()  # it is also stop of training time
    #for feat_lab_tuple, path in HDFDatasetDataProviderUtt(devel_files_list, valid_dataset, randomize=False, max_utt=-10):
    #    features, labels = feat_lab_tuple

    tqueue = TNetsCacheSimple.make_queue()
    cache = TNetsCacheSimple(tqueue, offset=0, num_batches_per_bunch=512)

    #cache.deamon = True
    cache.data_provider = dataprovider_test
    cache.start()

    #ex_num = 0

    while True:

        feats_lab_tuple = TNetsCacheSimple.get_elem_from_queue(tqueue)
        if isinstance(feats_lab_tuple, TNetsCacheLastElem):
            break

        features, labels = feats_lab_tuple

        test_frames_showed += features.shape[0]
        #test_set_x.set_value(features, borrow=True)
        #test_set_y.set_value(numpy.asarray(labels.flatten(), 'int32'), borrow=True)

        test_batches = features.shape[0] / batch_size
        #print valid_batches
        #if there is any part left in utterance (smaller than a batch_size), take it into account at the end
        if (features.shape[0] % batch_size != 0
                or features.shape[0] < batch_size):
            test_batches += 1

    #for i in xrange(test_batches):
        for i in xrange(features.shape[0]):
            temp_features = []
            for j in features[i]:
                temp_features.append(
                    classifier.projectionlayer.word_rep.get_value(
                        borrow=True)[j])
            test_set_x.set_value(numpy.asarray(temp_features,
                                               dtype='float32').flatten(),
                                 borrow=True)
            test_set_y.set_value(numpy.asarray([labels[i]], dtype='int32'),
                                 borrow=True)
            log_likelihood.append(test_model())

        progress += 1
        if progress % 100 == 0:
            end_time_test_progress = time.time()
            print 'PROGRESS: Processed %i bunches (%i frames),  TIME: %f in seconds'\
                           %(progress, test_frames_showed, end_time_test_progress - start_test_time)

    end_time_test_progress = time.time()
    print 'PROGRESS: Processed %i bunches (%i frames),  TIME: %f in seconds'\
                    %(progress, test_frames_showed, end_time_test_progress - start_test_time)
    test_set_x.set_value(numpy.empty((1), 'float32'))
    test_set_y.set_value(numpy.empty((1), 'int32'))

    print numpy.sum(log_likelihood)
    likelihood_sum = (-numpy.sum(log_likelihood) / test_frames_showed)
    print 'likelihood_sum', likelihood_sum