outputtype = config['outtype'] #hinge or softmax leftsize = int(config['left']) rightsize = int(config['right']) sum_window = int(config['sumwindow']) label_type = config['target'] #nt or ct max_window = int(config['maxwindow']) slotposition = max_window / 2 logger.info('slotpostion is %d', slotposition) (typeIndMap, n_targets, wordvecs, vectorsize, typefreq_traindev) = utils.loadTypesAndVectors(targetTypesFile, vectorFile) (traincontexts, resultVectorTrain, resultVectorTrainAll) = utils.read_lines_data(trainfile, typeIndMap, max_window, label_type, -1) logger.info("number of training examples: %d", len(traincontexts)) inputMatrixTrain = utils.adeltheanomatrix_flexible(slotposition, vectorsize, traincontexts, wordvecs, leftsize, rightsize, sum_window, useSum) (contextlistDev, resultVectorDev, resultVectorDevAll) = utils.read_lines_data(devfile, typeIndMap, max_window, label_type, -1) logger.info("number of validation examples: %d", len(contextlistDev)) inputMatrixDev = utils.adeltheanomatrix_flexible(slotposition, vectorsize, contextlistDev, wordvecs, leftsize, rightsize, sum_window, useSum)
outputtype = config['outtype'] #hinge or softmax leftsize = int(config['left']) rightsize = int(config['right']) sum_window = int(config['sumwindow']) max_window = int(config['maxwindow']) slotposition = max_window / 2 label_type = config['target'] #nt or ct (typeIndMap, n_targets, wordvecs, vectorsize, typefreq_traindev) = utils.loadTypesAndVectors(targetTypesFile, vectorFile, -1) logger.info('word2vec vectors are loaded') (testcontexts, resultVectorTest, resultVectorTestAll) = utils.read_lines_data(testsetfile, typeIndMap, max_window, label_type, -1) logger.info("number of test examples: %d", len(testcontexts)) inputMatrixTest = utils.adeltheanomatrix_flexible(slotposition, vectorsize, testcontexts, wordvecs, leftsize, rightsize, sum_window, useSum) contextsize = leftsize + rightsize + 1 ################# for memory ############ testcontexts = [] wordvecs = [] ##################### the network ####################### test_set_x = theano.shared( numpy.matrix(inputMatrixTest, dtype=theano.config.floatX)) # @UndefinedVariable test_set_y = theano.shared(
use_tanh_out = False if 'tanh' in config: use_tanh_out = True outputtype = config['outtype'] #hinge or softmax leftsize = int(config['left']) rightsize = int(config['right']) sum_window = int(config['sumwindow']) max_window = int(config['maxwindow']) slotposition = max_window / 2 label_type = config['target'] #nt or ct (typeIndMap, n_targets, wordvecs, vectorsize, typefreq_traindev) = utils.loadTypesAndVectors(targetTypesFile, vectorFile, -1) logger.info('word2vec vectors are loaded') (testcontexts,resultVectorTest, resultVectorTestAll) = utils.read_lines_data(testsetfile, typeIndMap, max_window, label_type, -1) logger.info("number of test examples: %d", len(testcontexts)) inputMatrixTest = utils.adeltheanomatrix_flexible(slotposition, vectorsize, testcontexts, wordvecs, leftsize, rightsize, sum_window, useSum) contextsize = leftsize + rightsize + 1 ################# for memory ############ testcontexts = []; wordvecs = []; ##################### the network ####################### test_set_x = theano.shared(numpy.matrix(inputMatrixTest, dtype=theano.config.floatX)) # @UndefinedVariable test_set_y = theano.shared(numpy.matrix(resultVectorTestAll, dtype=numpy.dtype(numpy.int32))) rng = numpy.random.RandomState(23455) n_test_batches = test_set_x.get_value(borrow=True).shape[0] n_test_batches /= batch_size
if 'tanh' in config: use_tanh_out = True outputtype = config['outtype'] #hinge or softmax leftsize = int(config['left']) rightsize = int(config['right']) sum_window = int(config['sumwindow']) label_type = config['target'] #nt or ct max_window = int(config['maxwindow']) slotposition = max_window / 2 logger.info('slotpostion is %d', slotposition) (typeIndMap, n_targets, wordvecs, vectorsize, typefreq_traindev) = utils.loadTypesAndVectors(targetTypesFile, vectorFile) (traincontexts,resultVectorTrain, resultVectorTrainAll) = utils.read_lines_data(trainfile, typeIndMap, max_window, label_type, -1) logger.info("number of training examples: %d", len(traincontexts)) inputMatrixTrain = utils.adeltheanomatrix_flexible(slotposition, vectorsize, traincontexts, wordvecs, leftsize, rightsize, sum_window, useSum) (contextlistDev,resultVectorDev,resultVectorDevAll) = utils.read_lines_data(devfile, typeIndMap, max_window, label_type, -1) logger.info("number of validation examples: %d", len(contextlistDev)) inputMatrixDev = utils.adeltheanomatrix_flexible(slotposition, vectorsize, contextlistDev, wordvecs, leftsize, rightsize, sum_window, useSum) contextsize = leftsize + rightsize + 1 #build negative results for hinge loss resultMatrixDevAll = numpy.empty(shape=(len(resultVectorDevAll), n_targets)) for i in xrange(len(resultVectorDevAll)): for j in range(n_targets):