def main(args):
np.random.seed(0xC0FFEE)
train, test, dicts = pkl.load( open('datas/atis.pkl', 'r') )
index2words = {value:key for key, value in dicts['words2idx'].iteritems()}
index2tables = {value:key for key, value in dicts['tables2idx'].iteritems()}
index2labels = {value:key for key, value in dicts['labels2idx'].iteritems()}
train_lex, train_ne, train_y = train
test_lex, test_ne, test_y = test
vocsize = len(dicts['words2idx']) + 1
nclasses = len(dicts['labels2idx'])
nsentences = len(train_lex)
context_window_size = 7
learning_rate = 0.01
n = Network()
n.layers.append( Fullconnect(vocsize, 100, Tanh.function, Tanh.derivative, updater=GradientDescent(learning_rate)) )
n.layers.append( BiRecurrent(100, 100, Tanh.function, Tanh.derivative, updater=GradientDescent(learning_rate)) )
n.layers.append( Fullconnect(100, nclasses, updater=GradientDescent(learning_rate)) )
n.activation = Softmax()
for epoch in range(0, 11):
epoch_loss = 0
for i in xrange(nsentences):
cwords = contextwin(train_lex[i], context_window_size)
words, labels = onehotvector(cwords, vocsize, train_y[i], nclasses)
loss = n.train( words, labels ) / len(words) # sequence normalized loss
epoch_loss += loss
if i%1000 == 0:
logging.info( 'epoch:%04d iter:%04d loss:%.2f'%(epoch, i, epoch_loss/(i+1)) )
logging.info( 'epoch:%04d loss:%.2f'%(epoch, epoch_loss/nsentences) )
for i in range(20):
idx = random.randint(0, len(test_lex)-1)
cwords = contextwin(test_lex[idx], context_window_size)
words = onehotvector(cwords, vocsize)[0]
labels = test_y[idx]
_ = n.predict(words)
y = [np.argmax(prediction) for prediction in _]
#print _
#print y
print 'word: ', ' '.join([index2words[_] for _ in test_lex[idx]])
print 'label: ', ' '.join([index2labels[_] for _ in labels])
print 'predict:', ' '.join([index2labels[_] for _ in y])
def main(args):
logging.info("load data start")
train_lex, train_y = pkl.load(open("datas/kowiki_spacing_train.pkl", "r"))
words2idx = pkl.load(open("datas/kowiki_dict.pkl", "r"))
logging.info("load data done")
index2words = {value: key for key, value in words2idx.iteritems()}
vocsize = len(words2idx) + 1
nclasses = 2
nsentences = len(train_lex)
max_iter = min(args.samples, nsentences)
logging.info(
"vocsize:%d, nclasses:%d, nsentences:%d, samples:%d, max_iter:%d"
% (vocsize, nclasses, nsentences, args.samples, max_iter)
)
context_window_size = args.window_size
n = Network()
n.layers.append(Fullconnect(vocsize, 256, Tanh.function))
n.layers.append(Recurrent(256, 256, ReLU.function))
n.layers.append(Fullconnect(256, 256, ReLU.function))
n.layers.append(Fullconnect(256, nclasses))
n.activation = Softmax(is_zero_pad=True)
if not os.path.isfile(args.params):
logging.error("not exist parameter file: %s" % args.params)
return
n.load_params(pkl.load(open(args.params, "rb")))
for i in xrange(max_iter):
cwords = contextwin(train_lex[i], context_window_size)
words, labels = onehotvector(cwords, vocsize)
y_list = [np.argmax(_) for _ in n.predict(words)]
result_list = []
for idx, y in zip(train_lex[i], y_list):
if y == 1:
result_list.append(" ")
result_list.append(index2words[idx].encode("utf8"))
print "".join(result_list)
def main(args):
logging.info('load data start')
train_lex, train_y = pkl.load(open('datas/kowiki_spacing_train.pkl', 'r'))
words2idx = pkl.load(open('datas/kowiki_dict.pkl', 'r'))
logging.info('load data done')
index2words = {value: key for key, value in words2idx.iteritems()}
vocsize = len(words2idx) + 1
nclasses = 2
nsentences = len(train_lex)
max_iter = min(args.samples, nsentences)
logging.info(
'vocsize:%d, nclasses:%d, nsentences:%d, samples:%d, max_iter:%d' %
(vocsize, nclasses, nsentences, args.samples, max_iter))
context_window_size = args.window_size
n = Network()
n.layers.append(Fullconnect(vocsize, 256, Tanh.function))
n.layers.append(Recurrent(256, 256, ReLU.function))
n.layers.append(Fullconnect(256, 256, ReLU.function))
n.layers.append(Fullconnect(256, nclasses))
n.activation = Softmax(is_zero_pad=True)
if not os.path.isfile(args.params):
logging.error('not exist parameter file: %s' % args.params)
return
n.load_params(pkl.load(open(args.params, 'rb')))
for i in xrange(max_iter):
cwords = contextwin(train_lex[i], context_window_size)
words, labels = onehotvector(cwords, vocsize)
y_list = [np.argmax(_) for _ in n.predict(words)]
result_list = []
for idx, y in zip(train_lex[i], y_list):
if y == 1:
result_list.append(' ')
result_list.append(index2words[idx].encode('utf8'))
print ''.join(result_list)
def main(args):
np.random.seed(0xC0FFEE)
train, test, dicts = pkl.load(open('datas/atis.pkl', 'r'))
index2words = {value: key for key, value in dicts['words2idx'].iteritems()}
index2tables = {
value: key
for key, value in dicts['tables2idx'].iteritems()
}
index2labels = {
value: key
for key, value in dicts['labels2idx'].iteritems()
}
datas = [
{
'name': 'train',
'x': train[0],
'y': train[2],
'size': len(train[0])
},
{
'name': 'test',
'x': test[0],
'y': test[2],
'size': len(test[0])
},
]
vocsize = len(dicts['words2idx']) + 1
nclasses = len(dicts['labels2idx'])
context_window_size = args.window_size
n = Network()
# word embedding layer
n.layers.append(Fullconnect(vocsize, 256, Tanh.function, Tanh.derivative))
# recurrent layer
n.layers.append(
Recurrent(n.layers[-1].output_size, 256, ReLU.function,
ReLU.derivative))
n.layers.append(
Dropout(n.layers[-1].output_size, 256, 0.5, ReLU.function,
ReLU.derivative))
n.layers.append(Fullconnect(n.layers[-1].output_size, nclasses))
n.activation = Softmax(is_zero_pad=True)
if not os.path.isfile(args.params):
logging.error('not exist params: %s' % args.params)
return
fname = args.params
n.load_params(pkl.load(open(fname, 'rb')))
logging.info('load parameters at %s' % (fname))
# prediction setup for evaluation
for l, layer in enumerate(n.layers):
if 'Dropout' == type(layer).__name__:
n.layers[l].is_testing = True
data = datas[1]
max_iteration = data['size']
results = {'p': [], 'g': [], 'w': []}
for i in range(max_iteration):
idx = i
x = data['x'][idx]
labels = data['y'][idx]
cwords = contextwin(datas[1]['x'][idx], context_window_size)
words = onehotvector(cwords, vocsize)[0]
_ = n.predict(words)
y = [np.argmax(prediction) for prediction in _]
results['p'].append([index2tables[_] for _ in y])
results['g'].append([index2tables[_] for _ in labels])
results['w'].append([index2words[_] for _ in x])
rv = conlleval(results['p'], results['g'], results['w'],
'atis_test_file.tmp')
logging.info('evaluation result: %s' % (str(rv)))
for i in range(20):
idx = random.randint(0, datas[1]['size'] - 1)
x = datas[1]['x'][idx]
labels = datas[1]['y'][idx]
cwords = contextwin(datas[1]['x'][idx], context_window_size)
words = onehotvector(cwords, vocsize)[0]
_ = n.predict(words)
y = [np.argmax(prediction) for prediction in _]
print 'word: ', ' '.join([index2words[_] for _ in x])
print 'table: ', ' '.join([index2tables[_] for _ in labels])
print 'label: ', ' '.join([index2labels[_] for _ in labels])
print 'predict:', ' '.join([index2labels[_] for _ in y])
def main(args):
np.random.seed(0xC0FFEE)
logging.info('load data start')
train_lex, train_y = pkl.load(open('datas/kowiki_spacing_train.pkl', 'r'))
words2idx = pkl.load(open('datas/kowiki_dict.pkl', 'r'))
logging.info('load data done')
index2words = {value: key for key, value in words2idx.iteritems()}
vocsize = len(words2idx) + 1
nclasses = 2
nsentences = len(train_lex)
context_window_size = args.window_size
minibatch = args.minibatch
learning_rate = args.learning_rate
logging.info(
'vocsize:%d, nclasses:%d, window-size:%d, minibatch:%d, learning-rate:%.5f'
% (vocsize, nclasses, context_window_size, minibatch, learning_rate))
n = Network()
n.layers.append(
Fullconnect(vocsize,
256,
Tanh.function,
Tanh.derivative,
updater=GradientDescent(learning_rate)))
n.layers.append(
Recurrent(256,
256,
ReLU.function,
ReLU.derivative,
updater=GradientDescent(learning_rate)))
n.layers.append(
Fullconnect(256,
256,
ReLU.function,
ReLU.derivative,
updater=GradientDescent(learning_rate)))
n.layers.append(
Fullconnect(256, nclasses, updater=GradientDescent(learning_rate)))
n.activation = Softmax(is_zero_pad=True)
if os.path.isfile(args.params):
logging.info('load parameters from %s' % args.params)
n.load_params(pkl.load(open(args.params, 'rb')))
logging.info('train start')
for epoch in xrange(0, args.epoch):
epoch_loss = 0
epoch_error_rate = 0
max_iterations = min(args.samples, nsentences) / minibatch
for i in xrange(max_iterations):
max_size_of_sequence = 100
idxs = [
random.randint(0, nsentences - 1) for _ in range(minibatch)
]
cwords = [
contextwin(train_lex[idx][:max_size_of_sequence],
context_window_size) for idx in idxs
]
words_labels = [
onehotvector(cword, vocsize,
train_y[idx][:max_size_of_sequence], nclasses)
for idx, cword in zip(idxs, cwords)
]
words = [word for word, label in words_labels]
labels = [label for word, label in words_labels]
# zero padding for minibatch
max_size_of_sequence = max([_.shape[0] for _ in words])
for k, (word, label) in enumerate(zip(words, labels)):
size_of_sequence = word.shape[0]
words[k] = np.pad(
word,
((0, max_size_of_sequence - size_of_sequence), (0, 0)),
mode='constant')
labels[k] = np.pad(
label,
((0, max_size_of_sequence - size_of_sequence), (0, 0)),
mode='constant')
words = np.swapaxes(np.array(words), 0, 1)
labels = np.swapaxes(np.array(labels), 0, 1)
loss = n.train(words, labels) / (max_size_of_sequence * minibatch
) # sequence normalized loss
predictions = n.y
error_rate = n.activation.error(
predictions, labels) / (max_size_of_sequence * minibatch)
epoch_loss += loss
epoch_error_rate += error_rate
if i % 10 == 0 and i != 0:
logging.info(
'[%.4f%%] epoch:%04d iter:%04d loss:%.5f error-rate:%.5f' %
((i + 1) / float(max_iterations), epoch, i, epoch_loss /
(i + 1), epoch_error_rate / (i + 1)))
logging.info('epoch:%04d loss:%.5f, error-rate:%.5f' %
(epoch, epoch_loss / max_iterations,
epoch_error_rate / max_iterations))
pkl.dump(n.dump_params(), open(args.params, 'wb'))
logging.info('dump parameters at %s' % (args.params))
def main(args):
np.random.seed(0xC0FFEE)
logging.info('load data start')
train_lex, train_y = pkl.load( open('datas/kowiki_spacing_train.pkl', 'r') )
words2idx = pkl.load( open('datas/kowiki_dict.pkl', 'r') )
logging.info('load data done')
index2words = {value:key for key, value in words2idx.iteritems()}
vocsize = len(words2idx) + 1
nclasses = 2
nsentences = len(train_lex)
context_window_size = args.window_size
minibatch = args.minibatch
learning_rate = args.learning_rate
logging.info('vocsize:%d, nclasses:%d, window-size:%d, minibatch:%d, learning-rate:%.5f'%(vocsize, nclasses, context_window_size, minibatch, learning_rate))
n = Network()
n.layers.append( Fullconnect(vocsize, 256, Tanh.function, Tanh.derivative, updater=GradientDescent(learning_rate)) )
n.layers.append( Recurrent(256, 256, ReLU.function, ReLU.derivative, updater=GradientDescent(learning_rate)) )
n.layers.append( Fullconnect(256, 256, ReLU.function, ReLU.derivative, updater=GradientDescent(learning_rate)) )
n.layers.append( Fullconnect(256, nclasses, updater=GradientDescent(learning_rate)) )
n.activation = Softmax(is_zero_pad=True)
if os.path.isfile( args.params ):
logging.info('load parameters from %s'%args.params)
n.load_params( pkl.load(open(args.params, 'rb')) )
logging.info('train start')
for epoch in xrange(0, args.epoch):
epoch_loss = 0
epoch_error_rate = 0
max_iterations = min(args.samples, nsentences) / minibatch
for i in xrange( max_iterations ):
max_size_of_sequence = 100
idxs = [random.randint(0, nsentences-1) for _ in range(minibatch)]
cwords = [contextwin(train_lex[idx][:max_size_of_sequence], context_window_size) for idx in idxs]
words_labels = [onehotvector(cword, vocsize, train_y[idx][:max_size_of_sequence], nclasses) for idx, cword in zip(idxs, cwords)]
words = [word for word, label in words_labels]
labels = [label for word, label in words_labels]
# zero padding for minibatch
max_size_of_sequence = max( [_.shape[0] for _ in words] )
for k, (word, label) in enumerate(zip(words, labels)):
size_of_sequence = word.shape[0]
words[k] = np.pad(word, ((0, max_size_of_sequence-size_of_sequence), (0, 0)), mode='constant')
labels[k] = np.pad(label, ((0, max_size_of_sequence-size_of_sequence), (0, 0)), mode='constant')
words = np.swapaxes( np.array(words), 0, 1 )
labels = np.swapaxes( np.array(labels), 0, 1 )
loss = n.train( words, labels ) / (max_size_of_sequence * minibatch) # sequence normalized loss
predictions = n.y
error_rate = n.activation.error( predictions, labels ) / (max_size_of_sequence * minibatch)
epoch_loss += loss
epoch_error_rate += error_rate
if i%10 == 0 and i != 0:
logging.info('[%.4f%%] epoch:%04d iter:%04d loss:%.5f error-rate:%.5f'%((i+1)/float(max_iterations), epoch, i, epoch_loss/(i+1), epoch_error_rate/(i+1)))
logging.info('epoch:%04d loss:%.5f, error-rate:%.5f'%(epoch, epoch_loss/max_iterations, epoch_error_rate/max_iterations))
pkl.dump( n.dump_params(), open(args.params, 'wb') )
logging.info('dump parameters at %s'%(args.params))
def main(args):
np.random.seed(0xC0FFEE)
train, test, dicts = pkl.load( open('datas/atis.pkl', 'r') )
index2words = {value:key for key, value in dicts['words2idx'].iteritems()}
index2tables = {value:key for key, value in dicts['tables2idx'].iteritems()}
index2labels = {value:key for key, value in dicts['labels2idx'].iteritems()}
datas = [
{'name':'train', 'x':train[0], 'y':train[2], 'size':len(train[0])},
{'name':'test', 'x':test[0], 'y':test[2], 'size':len(test[0])},
]
vocsize = len(dicts['words2idx']) + 1
nclasses = len(dicts['labels2idx'])
context_window_size = args.window_size
n = Network()
# word embedding layer
n.layers.append( Fullconnect(vocsize, 256, Tanh.function, Tanh.derivative) )
# recurrent layer
n.layers.append( Recurrent(n.layers[-1].output_size, 256, ReLU.function, ReLU.derivative) )
n.layers.append( Dropout(n.layers[-1].output_size, 256, 0.5, ReLU.function, ReLU.derivative) )
n.layers.append( Fullconnect(n.layers[-1].output_size, nclasses) )
n.activation = Softmax(is_zero_pad=True)
if not os.path.isfile( args.params ):
logging.error('not exist params: %s'%args.params)
return
fname = args.params
n.load_params( pkl.load( open(fname, 'rb') ) )
logging.info('load parameters at %s'%(fname))
# prediction setup for evaluation
for l, layer in enumerate(n.layers):
if 'Dropout' == type( layer ).__name__:
n.layers[l].is_testing = True
data = datas[1]
max_iteration = data['size']
results = {'p':[], 'g':[], 'w':[]}
for i in range(max_iteration):
idx = i
x = data['x'][idx]
labels = data['y'][idx]
cwords = contextwin(datas[1]['x'][idx], context_window_size)
words = onehotvector(cwords, vocsize)[0]
_ = n.predict(words)
y = [np.argmax(prediction) for prediction in _]
results['p'].append( [index2tables[_] for _ in y] )
results['g'].append( [index2tables[_] for _ in labels] )
results['w'].append( [index2words[_] for _ in x] )
rv = conlleval(results['p'], results['g'], results['w'], 'atis_test_file.tmp')
logging.info('evaluation result: %s'%(str(rv)))
for i in range(20):
idx = random.randint(0, datas[1]['size']-1)
x = datas[1]['x'][idx]
labels = datas[1]['y'][idx]
cwords = contextwin(datas[1]['x'][idx], context_window_size)
words = onehotvector(cwords, vocsize)[0]
_ = n.predict(words)
y = [np.argmax(prediction) for prediction in _]
print 'word: ', ' '.join([index2words[_] for _ in x])
print 'table: ', ' '.join([index2tables[_] for _ in labels])
print 'label: ', ' '.join([index2labels[_] for _ in labels])
print 'predict:', ' '.join([index2labels[_] for _ in y])
