def main(args): np.random.seed(0xC0FFEE) train, test, dicts = pkl.load( open('datas/atis.pkl', 'r') ) index2words = {value:key for key, value in dicts['words2idx'].iteritems()} index2tables = {value:key for key, value in dicts['tables2idx'].iteritems()} index2labels = {value:key for key, value in dicts['labels2idx'].iteritems()} train_lex, train_ne, train_y = train test_lex, test_ne, test_y = test vocsize = len(dicts['words2idx']) + 1 nclasses = len(dicts['labels2idx']) nsentences = len(train_lex) context_window_size = 7 learning_rate = 0.01 n = Network() n.layers.append( Fullconnect(vocsize, 100, Tanh.function, Tanh.derivative, updater=GradientDescent(learning_rate)) ) n.layers.append( BiRecurrent(100, 100, Tanh.function, Tanh.derivative, updater=GradientDescent(learning_rate)) ) n.layers.append( Fullconnect(100, nclasses, updater=GradientDescent(learning_rate)) ) n.activation = Softmax() for epoch in range(0, 11): epoch_loss = 0 for i in xrange(nsentences): cwords = contextwin(train_lex[i], context_window_size) words, labels = onehotvector(cwords, vocsize, train_y[i], nclasses) loss = n.train( words, labels ) / len(words) # sequence normalized loss epoch_loss += loss if i%1000 == 0: logging.info( 'epoch:%04d iter:%04d loss:%.2f'%(epoch, i, epoch_loss/(i+1)) ) logging.info( 'epoch:%04d loss:%.2f'%(epoch, epoch_loss/nsentences) ) for i in range(20): idx = random.randint(0, len(test_lex)-1) cwords = contextwin(test_lex[idx], context_window_size) words = onehotvector(cwords, vocsize)[0] labels = test_y[idx] _ = n.predict(words) y = [np.argmax(prediction) for prediction in _] #print _ #print y print 'word: ', ' '.join([index2words[_] for _ in test_lex[idx]]) print 'label: ', ' '.join([index2labels[_] for _ in labels]) print 'predict:', ' '.join([index2labels[_] for _ in y])
def main(args): logging.info("load data start") train_lex, train_y = pkl.load(open("datas/kowiki_spacing_train.pkl", "r")) words2idx = pkl.load(open("datas/kowiki_dict.pkl", "r")) logging.info("load data done") index2words = {value: key for key, value in words2idx.iteritems()} vocsize = len(words2idx) + 1 nclasses = 2 nsentences = len(train_lex) max_iter = min(args.samples, nsentences) logging.info( "vocsize:%d, nclasses:%d, nsentences:%d, samples:%d, max_iter:%d" % (vocsize, nclasses, nsentences, args.samples, max_iter) ) context_window_size = args.window_size n = Network() n.layers.append(Fullconnect(vocsize, 256, Tanh.function)) n.layers.append(Recurrent(256, 256, ReLU.function)) n.layers.append(Fullconnect(256, 256, ReLU.function)) n.layers.append(Fullconnect(256, nclasses)) n.activation = Softmax(is_zero_pad=True) if not os.path.isfile(args.params): logging.error("not exist parameter file: %s" % args.params) return n.load_params(pkl.load(open(args.params, "rb"))) for i in xrange(max_iter): cwords = contextwin(train_lex[i], context_window_size) words, labels = onehotvector(cwords, vocsize) y_list = [np.argmax(_) for _ in n.predict(words)] result_list = [] for idx, y in zip(train_lex[i], y_list): if y == 1: result_list.append(" ") result_list.append(index2words[idx].encode("utf8")) print "".join(result_list)
def main(args): logging.info('load data start') train_lex, train_y = pkl.load(open('datas/kowiki_spacing_train.pkl', 'r')) words2idx = pkl.load(open('datas/kowiki_dict.pkl', 'r')) logging.info('load data done') index2words = {value: key for key, value in words2idx.iteritems()} vocsize = len(words2idx) + 1 nclasses = 2 nsentences = len(train_lex) max_iter = min(args.samples, nsentences) logging.info( 'vocsize:%d, nclasses:%d, nsentences:%d, samples:%d, max_iter:%d' % (vocsize, nclasses, nsentences, args.samples, max_iter)) context_window_size = args.window_size n = Network() n.layers.append(Fullconnect(vocsize, 256, Tanh.function)) n.layers.append(Recurrent(256, 256, ReLU.function)) n.layers.append(Fullconnect(256, 256, ReLU.function)) n.layers.append(Fullconnect(256, nclasses)) n.activation = Softmax(is_zero_pad=True) if not os.path.isfile(args.params): logging.error('not exist parameter file: %s' % args.params) return n.load_params(pkl.load(open(args.params, 'rb'))) for i in xrange(max_iter): cwords = contextwin(train_lex[i], context_window_size) words, labels = onehotvector(cwords, vocsize) y_list = [np.argmax(_) for _ in n.predict(words)] result_list = [] for idx, y in zip(train_lex[i], y_list): if y == 1: result_list.append(' ') result_list.append(index2words[idx].encode('utf8')) print ''.join(result_list)
def main(args): np.random.seed(0xC0FFEE) train, test, dicts = pkl.load(open('datas/atis.pkl', 'r')) index2words = {value: key for key, value in dicts['words2idx'].iteritems()} index2tables = { value: key for key, value in dicts['tables2idx'].iteritems() } index2labels = { value: key for key, value in dicts['labels2idx'].iteritems() } datas = [ { 'name': 'train', 'x': train[0], 'y': train[2], 'size': len(train[0]) }, { 'name': 'test', 'x': test[0], 'y': test[2], 'size': len(test[0]) }, ] vocsize = len(dicts['words2idx']) + 1 nclasses = len(dicts['labels2idx']) context_window_size = args.window_size n = Network() # word embedding layer n.layers.append(Fullconnect(vocsize, 256, Tanh.function, Tanh.derivative)) # recurrent layer n.layers.append( Recurrent(n.layers[-1].output_size, 256, ReLU.function, ReLU.derivative)) n.layers.append( Dropout(n.layers[-1].output_size, 256, 0.5, ReLU.function, ReLU.derivative)) n.layers.append(Fullconnect(n.layers[-1].output_size, nclasses)) n.activation = Softmax(is_zero_pad=True) if not os.path.isfile(args.params): logging.error('not exist params: %s' % args.params) return fname = args.params n.load_params(pkl.load(open(fname, 'rb'))) logging.info('load parameters at %s' % (fname)) # prediction setup for evaluation for l, layer in enumerate(n.layers): if 'Dropout' == type(layer).__name__: n.layers[l].is_testing = True data = datas[1] max_iteration = data['size'] results = {'p': [], 'g': [], 'w': []} for i in range(max_iteration): idx = i x = data['x'][idx] labels = data['y'][idx] cwords = contextwin(datas[1]['x'][idx], context_window_size) words = onehotvector(cwords, vocsize)[0] _ = n.predict(words) y = [np.argmax(prediction) for prediction in _] results['p'].append([index2tables[_] for _ in y]) results['g'].append([index2tables[_] for _ in labels]) results['w'].append([index2words[_] for _ in x]) rv = conlleval(results['p'], results['g'], results['w'], 'atis_test_file.tmp') logging.info('evaluation result: %s' % (str(rv))) for i in range(20): idx = random.randint(0, datas[1]['size'] - 1) x = datas[1]['x'][idx] labels = datas[1]['y'][idx] cwords = contextwin(datas[1]['x'][idx], context_window_size) words = onehotvector(cwords, vocsize)[0] _ = n.predict(words) y = [np.argmax(prediction) for prediction in _] print 'word: ', ' '.join([index2words[_] for _ in x]) print 'table: ', ' '.join([index2tables[_] for _ in labels]) print 'label: ', ' '.join([index2labels[_] for _ in labels]) print 'predict:', ' '.join([index2labels[_] for _ in y])
def main(args): np.random.seed(0xC0FFEE) logging.info('load data start') train_lex, train_y = pkl.load(open('datas/kowiki_spacing_train.pkl', 'r')) words2idx = pkl.load(open('datas/kowiki_dict.pkl', 'r')) logging.info('load data done') index2words = {value: key for key, value in words2idx.iteritems()} vocsize = len(words2idx) + 1 nclasses = 2 nsentences = len(train_lex) context_window_size = args.window_size minibatch = args.minibatch learning_rate = args.learning_rate logging.info( 'vocsize:%d, nclasses:%d, window-size:%d, minibatch:%d, learning-rate:%.5f' % (vocsize, nclasses, context_window_size, minibatch, learning_rate)) n = Network() n.layers.append( Fullconnect(vocsize, 256, Tanh.function, Tanh.derivative, updater=GradientDescent(learning_rate))) n.layers.append( Recurrent(256, 256, ReLU.function, ReLU.derivative, updater=GradientDescent(learning_rate))) n.layers.append( Fullconnect(256, 256, ReLU.function, ReLU.derivative, updater=GradientDescent(learning_rate))) n.layers.append( Fullconnect(256, nclasses, updater=GradientDescent(learning_rate))) n.activation = Softmax(is_zero_pad=True) if os.path.isfile(args.params): logging.info('load parameters from %s' % args.params) n.load_params(pkl.load(open(args.params, 'rb'))) logging.info('train start') for epoch in xrange(0, args.epoch): epoch_loss = 0 epoch_error_rate = 0 max_iterations = min(args.samples, nsentences) / minibatch for i in xrange(max_iterations): max_size_of_sequence = 100 idxs = [ random.randint(0, nsentences - 1) for _ in range(minibatch) ] cwords = [ contextwin(train_lex[idx][:max_size_of_sequence], context_window_size) for idx in idxs ] words_labels = [ onehotvector(cword, vocsize, train_y[idx][:max_size_of_sequence], nclasses) for idx, cword in zip(idxs, cwords) ] words = [word for word, label in words_labels] labels = [label for word, label in words_labels] # zero padding for minibatch max_size_of_sequence = max([_.shape[0] for _ in words]) for k, (word, label) in enumerate(zip(words, labels)): size_of_sequence = word.shape[0] words[k] = np.pad( word, ((0, max_size_of_sequence - size_of_sequence), (0, 0)), mode='constant') labels[k] = np.pad( label, ((0, max_size_of_sequence - size_of_sequence), (0, 0)), mode='constant') words = np.swapaxes(np.array(words), 0, 1) labels = np.swapaxes(np.array(labels), 0, 1) loss = n.train(words, labels) / (max_size_of_sequence * minibatch ) # sequence normalized loss predictions = n.y error_rate = n.activation.error( predictions, labels) / (max_size_of_sequence * minibatch) epoch_loss += loss epoch_error_rate += error_rate if i % 10 == 0 and i != 0: logging.info( '[%.4f%%] epoch:%04d iter:%04d loss:%.5f error-rate:%.5f' % ((i + 1) / float(max_iterations), epoch, i, epoch_loss / (i + 1), epoch_error_rate / (i + 1))) logging.info('epoch:%04d loss:%.5f, error-rate:%.5f' % (epoch, epoch_loss / max_iterations, epoch_error_rate / max_iterations)) pkl.dump(n.dump_params(), open(args.params, 'wb')) logging.info('dump parameters at %s' % (args.params))
def main(args): np.random.seed(0xC0FFEE) logging.info('load data start') train_lex, train_y = pkl.load( open('datas/kowiki_spacing_train.pkl', 'r') ) words2idx = pkl.load( open('datas/kowiki_dict.pkl', 'r') ) logging.info('load data done') index2words = {value:key for key, value in words2idx.iteritems()} vocsize = len(words2idx) + 1 nclasses = 2 nsentences = len(train_lex) context_window_size = args.window_size minibatch = args.minibatch learning_rate = args.learning_rate logging.info('vocsize:%d, nclasses:%d, window-size:%d, minibatch:%d, learning-rate:%.5f'%(vocsize, nclasses, context_window_size, minibatch, learning_rate)) n = Network() n.layers.append( Fullconnect(vocsize, 256, Tanh.function, Tanh.derivative, updater=GradientDescent(learning_rate)) ) n.layers.append( Recurrent(256, 256, ReLU.function, ReLU.derivative, updater=GradientDescent(learning_rate)) ) n.layers.append( Fullconnect(256, 256, ReLU.function, ReLU.derivative, updater=GradientDescent(learning_rate)) ) n.layers.append( Fullconnect(256, nclasses, updater=GradientDescent(learning_rate)) ) n.activation = Softmax(is_zero_pad=True) if os.path.isfile( args.params ): logging.info('load parameters from %s'%args.params) n.load_params( pkl.load(open(args.params, 'rb')) ) logging.info('train start') for epoch in xrange(0, args.epoch): epoch_loss = 0 epoch_error_rate = 0 max_iterations = min(args.samples, nsentences) / minibatch for i in xrange( max_iterations ): max_size_of_sequence = 100 idxs = [random.randint(0, nsentences-1) for _ in range(minibatch)] cwords = [contextwin(train_lex[idx][:max_size_of_sequence], context_window_size) for idx in idxs] words_labels = [onehotvector(cword, vocsize, train_y[idx][:max_size_of_sequence], nclasses) for idx, cword in zip(idxs, cwords)] words = [word for word, label in words_labels] labels = [label for word, label in words_labels] # zero padding for minibatch max_size_of_sequence = max( [_.shape[0] for _ in words] ) for k, (word, label) in enumerate(zip(words, labels)): size_of_sequence = word.shape[0] words[k] = np.pad(word, ((0, max_size_of_sequence-size_of_sequence), (0, 0)), mode='constant') labels[k] = np.pad(label, ((0, max_size_of_sequence-size_of_sequence), (0, 0)), mode='constant') words = np.swapaxes( np.array(words), 0, 1 ) labels = np.swapaxes( np.array(labels), 0, 1 ) loss = n.train( words, labels ) / (max_size_of_sequence * minibatch) # sequence normalized loss predictions = n.y error_rate = n.activation.error( predictions, labels ) / (max_size_of_sequence * minibatch) epoch_loss += loss epoch_error_rate += error_rate if i%10 == 0 and i != 0: logging.info('[%.4f%%] epoch:%04d iter:%04d loss:%.5f error-rate:%.5f'%((i+1)/float(max_iterations), epoch, i, epoch_loss/(i+1), epoch_error_rate/(i+1))) logging.info('epoch:%04d loss:%.5f, error-rate:%.5f'%(epoch, epoch_loss/max_iterations, epoch_error_rate/max_iterations)) pkl.dump( n.dump_params(), open(args.params, 'wb') ) logging.info('dump parameters at %s'%(args.params))
def main(args): np.random.seed(0xC0FFEE) train, test, dicts = pkl.load( open('datas/atis.pkl', 'r') ) index2words = {value:key for key, value in dicts['words2idx'].iteritems()} index2tables = {value:key for key, value in dicts['tables2idx'].iteritems()} index2labels = {value:key for key, value in dicts['labels2idx'].iteritems()} datas = [ {'name':'train', 'x':train[0], 'y':train[2], 'size':len(train[0])}, {'name':'test', 'x':test[0], 'y':test[2], 'size':len(test[0])}, ] vocsize = len(dicts['words2idx']) + 1 nclasses = len(dicts['labels2idx']) context_window_size = args.window_size n = Network() # word embedding layer n.layers.append( Fullconnect(vocsize, 256, Tanh.function, Tanh.derivative) ) # recurrent layer n.layers.append( Recurrent(n.layers[-1].output_size, 256, ReLU.function, ReLU.derivative) ) n.layers.append( Dropout(n.layers[-1].output_size, 256, 0.5, ReLU.function, ReLU.derivative) ) n.layers.append( Fullconnect(n.layers[-1].output_size, nclasses) ) n.activation = Softmax(is_zero_pad=True) if not os.path.isfile( args.params ): logging.error('not exist params: %s'%args.params) return fname = args.params n.load_params( pkl.load( open(fname, 'rb') ) ) logging.info('load parameters at %s'%(fname)) # prediction setup for evaluation for l, layer in enumerate(n.layers): if 'Dropout' == type( layer ).__name__: n.layers[l].is_testing = True data = datas[1] max_iteration = data['size'] results = {'p':[], 'g':[], 'w':[]} for i in range(max_iteration): idx = i x = data['x'][idx] labels = data['y'][idx] cwords = contextwin(datas[1]['x'][idx], context_window_size) words = onehotvector(cwords, vocsize)[0] _ = n.predict(words) y = [np.argmax(prediction) for prediction in _] results['p'].append( [index2tables[_] for _ in y] ) results['g'].append( [index2tables[_] for _ in labels] ) results['w'].append( [index2words[_] for _ in x] ) rv = conlleval(results['p'], results['g'], results['w'], 'atis_test_file.tmp') logging.info('evaluation result: %s'%(str(rv))) for i in range(20): idx = random.randint(0, datas[1]['size']-1) x = datas[1]['x'][idx] labels = datas[1]['y'][idx] cwords = contextwin(datas[1]['x'][idx], context_window_size) words = onehotvector(cwords, vocsize)[0] _ = n.predict(words) y = [np.argmax(prediction) for prediction in _] print 'word: ', ' '.join([index2words[_] for _ in x]) print 'table: ', ' '.join([index2tables[_] for _ in labels]) print 'label: ', ' '.join([index2labels[_] for _ in labels]) print 'predict:', ' '.join([index2labels[_] for _ in y])