'lr': 0.0627142536696559,
'verbose': 1,
'decay': False, # decay on the learning rate if improvement stops
'win': 7, # number of words in the context window
'bs': 9, # number of backprop through time steps
'nhidden': 100, # number of hidden units
'seed': 345,
'emb_dimension': 100, # dimension of word embedding
'nepochs': 50
}
folder = os.path.basename(__file__).split('.')[0]
if not os.path.exists(folder): os.mkdir(folder)
# load the dataset
train_set, valid_set, test_set, dic = load.atisfold(s['fold'])
idx2label = dict((k, v) for v, k in dic['labels2idx'].iteritems())
idx2word = dict((k, v) for v, k in dic['words2idx'].iteritems())
train_lex, train_ne, train_y = train_set
valid_lex, valid_ne, valid_y = valid_set
test_lex, test_ne, test_y = test_set
vocsize = len(set(reduce( \
lambda x, y: list(x) + list(y), \
train_lex + valid_lex + test_lex)))
nclasses = len(set(reduce( \
lambda x, y: list(x) + list(y), \
train_y + test_y + valid_y)))
'lr': 0.0627142536696559,
'verbose': 1,
'decay': False, # decay on the learning rate if improvement stops
'win': 7, # number of words in the context window
'bs': 9, # number of backprop through time steps
'nhidden': 100, # number of hidden units
'seed': 345,
'emb_dimension': 100, # dimension of word embedding
'nepochs': 50
}
folder = os.path.basename(__file__).split('.')[0]
if not os.path.exists(folder): os.mkdir(folder)
# load the dataset
train_set, valid_set, test_set, dic = load.atisfold(args.input)
idx2label = dict((k, v) for v, k in dic['labels2idx'].iteritems())
idx2word = dict((k, v) for v, k in dic['words2idx'].iteritems())
train_lex, train_ne, train_y = train_set
valid_lex, valid_ne, valid_y = valid_set
test_lex, test_ne, test_y = test_set
vocsize = len(dic['words2idx'])
nclasses = len(dic['labels2idx'])
nsentences = len(train_lex)
# instanciate the model
numpy.random.seed(s['seed'])
random.seed(s['seed'])
rnn = model(nh=s['nhidden'],
if __name__ == '__main__':
folder = os.path.join(
'out/',
os.path.basename(__file__).split('.')[0]) # folder = 'out/bilstm-lm'
os.makedirs(folder, exist_ok=True)
print('ATIS fold:', FLAGS.fold)
print('with language model:', FLAGS.with_lm)
print('with GloVe:', FLAGS.with_glove)
print('with bi-LSTM:', FLAGS.bi_lstm)
print('Training epochs:', FLAGS.nepochs)
print('Training data size:', FLAGS.nsentences)
# load the dataset
train_set, valid_set, test_set, dic = load.atisfold(
FLAGS.fold) # size: 3983, 893, 893
idx2word = dict((k, v) for v, k in dic['words2idx'].items())
idx2label = dict((k, v) for v, k in dic['labels2idx'].items())
# id, named entity, label
train_x, train_ne, train_y = train_set
valid_x, valid_ne, valid_y = valid_set
test_x, test_ne, test_y = test_set
vocsize = len(dic['words2idx'])
nclasses = len(dic['labels2idx'])
nsentences = len(train_x)
assert FLAGS.nsentences <= nsentences, 'Training data size needs to be less than 3983.'
sentences_train = [
' '.join(list(map(lambda x: idx2word[x], s))) for s in train_x
s = {'fold':3, # 5 folds 0,1,2,3,4
'lr':0.0627142536696559,
'verbose':1,
'decay':False, # decay on the learning rate if improvement stops
'win':7, # number of words in the context window
'bs':9, # number of backprop through time steps
'nhidden':100, # number of hidden units
'seed':345,
'emb_dimension':100, # dimension of word embedding
'nepochs':50}
folder = os.path.basename(__file__).split('.')[0]
if not os.path.exists(folder): os.mkdir(folder)
# load the dataset
train_set, valid_set, test_set, dic = load.atisfold(s['fold'])
idx2label = dict((k,v) for v,k in dic['labels2idx'].iteritems())
idx2word = dict((k,v) for v,k in dic['words2idx'].iteritems())
train_lex, train_ne, train_y = train_set
valid_lex, valid_ne, valid_y = valid_set
test_lex, test_ne, test_y = test_set
vocsize = len(set(reduce(\
lambda x, y: list(x)+list(y),\
train_lex+valid_lex+test_lex)))
nclasses = len(set(reduce(\
lambda x, y: list(x)+list(y),\
train_y+test_y+valid_y)))
def test(s):
# load word vector
wv = np.load("./../WV/" + s['WVFolderName'] + "/" + s['model']+".words" + str(s['emb_dimension']) + ".npy")
# load vocab
with open("./../WV/" + s['WVFolderName'] + "/" + s['model']+".words" + str(s['emb_dimension']) + ".vocab") as f:
vocab = [line.strip() for line in f if len(line) > 0]
wi = dict([(a, i) for i, a in enumerate(vocab)])
iw = vocab
# load the dataset
if s['dataset'] == 'atis':
train_set, valid_set, test_set, dic = load.atisfold(s['fold'])
if s['dataset'] == 'ner':
train_set, valid_set, test_set, dic = load.ner()
if s['dataset'] == 'chunk':
train_set, valid_set, test_set, dic = load.chunk()
if s['dataset'] == 'pos':
train_set, valid_set, test_set, dic = load.pos()
idx2label = dict((k, v) for v, k in dic['labels2idx'].iteritems())
idx2word = dict((k, v) for v, k in dic['words2idx'].iteritems())
train_lex, train_ne, train_y = train_set
valid_lex, valid_ne, valid_y = valid_set
test_lex, test_ne, test_y = test_set
# train_lex.extend(valid_lex)
# train_ne.extend(valid_ne)
# train_y.extend(valid_y)
vocsize = len(dic['words2idx'])
nclasses = len(dic['labels2idx'])
my_train_input, my_train_y = getInputOutput(train_lex, train_y, s['win'], idx2word)
my_train_x = getX(my_train_input, wv, iw, wi)
my_test_input, my_test_y = getInputOutput(test_lex, test_y, s['win'], idx2word)
my_test_x = getX(my_test_input, wv, iw, wi)
clf = MLPClassifier(hidden_layer_sizes=(), verbose=False, activation='tanh')
clf.fit(my_train_x, my_train_y)
# eval
eval_options = []
if s['dataset'] == 'pos':
eval_options = ['-r']
my_train_yp = clf.predict(my_train_x)
my_test_yp = clf.predict(my_test_x)
# print my_train_y
# print my_train_yp
predictions_train = getFormatedPY(train_y, my_train_yp, idx2label)
groundtruth_train = [map(lambda x: idx2label[x], y) for y in train_y]
words_train = [map(lambda x: idx2word[x], w) for w in train_lex]
predictions_test = getFormatedPY(test_y, my_test_yp, idx2label)
groundtruth_test = [map(lambda x: idx2label[x], y) for y in test_y]
words_test = [map(lambda x: idx2word[x], w) for w in test_lex]
res_train = conlleval(predictions_train, groundtruth_train, words_train, folder + '/linear.train.' + s['dataset'] + '.txt', eval_options)
res_test = conlleval(predictions_test, groundtruth_test, words_test, folder + '/linear.test.' + s['dataset'] + '.txt', eval_options)
# print ' train', res_train['p'], res_train['r'], res_train['f1'] , ' ' * 20
# print ' test', res_test['p'], res_test['r'], res_test['f1'] , ' ' * 20
print res_test['f1'],
parser.add_argument('--seed', type=int, default=345, help='Seed')
parser.add_argument('--bs', type=int, default=9, help='Number of backprop through time steps')
parser.add_argument('--win', type=int, default=7, help='Number of words in context window')
parser.add_argument('--fold', type=int, default=4, help='Fold number, 0-4')
parser.add_argument('--lr', type=float, default=0.0627142536696559, help='Learning rate')
parser.add_argument('--verbose', type=int, default=1, help='Verbose or not')
parser.add_argument('--decay', type=int, default=0, help='Decay lr or not')
s = parser.parse_args()
print '*' * 80
print s
folder = os.path.basename(__file__).split('.')[0]
if not os.path.exists(folder): os.mkdir(folder)
# load the dataset
train_set, valid_set, test_set, dic = load.atisfold(s.fold)
idx2label = dict((k,v) for v,k in dic['labels2idx'].iteritems())
idx2word = dict((k,v) for v,k in dic['words2idx'].iteritems())
train_lex, train_ne, train_y = train_set
valid_lex, valid_ne, valid_y = valid_set
test_lex, test_ne, test_y = test_set
vocsize = len(set(reduce(\
lambda x, y: list(x)+list(y),\
train_lex+valid_lex+test_lex)))
nclasses = len(set(reduce(\
lambda x, y: list(x)+list(y),\
train_y+test_y+valid_y)))
"verbose": 1,
"decay": False, # decay on the learning rate if improvement stops
"win": 7, # number of words in the context window
"bs": 9, # number of backprop through time steps
"nhidden": 100, # number of hidden units
"seed": 345,
"emb_dimension": 100, # dimension of word embedding
"nepochs": 50,
}
folder = os.path.basename(__file__).split(".")[0]
if not os.path.exists(folder):
os.mkdir(folder)
# load the dataset
train_set, valid_set, test_set, dic = load.atisfold(s["fold"])
idx2label = dict((k, v) for v, k in dic["labels2idx"].iteritems())
idx2word = dict((k, v) for v, k in dic["words2idx"].iteritems())
train_lex, train_ne, train_y = train_set
valid_lex, valid_ne, valid_y = valid_set
test_lex, test_ne, test_y = test_set
vocsize = len(set(reduce(lambda x, y: list(x) + list(y), train_lex + valid_lex + test_lex)))
nclasses = len(set(reduce(lambda x, y: list(x) + list(y), train_y + test_y + valid_y)))
nsentences = len(train_lex)
# instanciate the model
numpy.random.seed(s["seed"])
default=0.0627142536696559,
help='Learning rate')
parser.add_argument('--verbose',
type=int,
default=1,
help='Verbose or not')
parser.add_argument('--decay', type=int, default=0, help='Decay lr or not')
s = parser.parse_args()
print '*' * 80
print s
folder = os.path.basename(__file__).split('.')[0]
if not os.path.exists(folder): os.mkdir(folder)
# load the dataset
train_set, valid_set, test_set, dic = load.atisfold(s.fold)
idx2label = dict((k, v) for v, k in dic['labels2idx'].iteritems())
idx2word = dict((k, v) for v, k in dic['words2idx'].iteritems())
train_lex, train_ne, train_y = train_set
valid_lex, valid_ne, valid_y = valid_set
test_lex, test_ne, test_y = test_set
vocsize = len(set(reduce(\
lambda x, y: list(x)+list(y),\
train_lex+valid_lex+test_lex)))
nclasses = len(set(reduce(\
lambda x, y: list(x)+list(y),\
train_y+test_y+valid_y)))
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-v', '--verbose', action='count', default=1,
help='Adjust level of verbosity.')
parser.add_argument('-nh', '--num-hidden', dest='num_hidden', type=int, default=100,
help='Set dimension of hidden units.')
parser.add_argument('-w', '--window', type=int, default=5,
help='Set size of context window (in words).')
parser.add_argument('-d', '--depth', type=int, default=3,
help='Set number of stacked layers')
parser.add_argument('--seed', type=int, default=345,
help='Set PRNG seed')
parser.add_argument('--emb-dim', dest='emb_dimension', type=int, default=100,
help='Set size of word embeddings')
parser.add_argument('-e', '--num-epochs', dest='num_epochs', type=int, default=50,
help='Set number of epochs to train')
args = parser.parse_args()
s = {'fold':3, # 5 folds 0,1,2,3,4
'lr':0.0627142536696559,
'verbose': args.verbose,
'decay': False, # decay on the learning rate if improvement stops
'win': args.window, # number of words in the context window
'bs':9, # number of backprop through time steps
'nhidden': args.num_hidden, # number of hidden units
'depth': args.depth, # number of layers in space
'seed': args.seed,
'emb_dimension': args.emb_dimension, # dimension of word embedding
'nepochs': args.num_epochs}
folder = os.path.basename(__file__).split('.')[0]
if not os.path.exists(folder): os.mkdir(folder)
# load the dataset
train_set, valid_set, test_set, dic = load.atisfold(s['fold'])
idx2label = dict((k,v) for v,k in dic['labels2idx'].iteritems())
idx2word = dict((k,v) for v,k in dic['words2idx'].iteritems())
train_lex, train_ne, train_y = train_set
valid_lex, valid_ne, valid_y = valid_set
test_lex, test_ne, test_y = test_set
vocsize = len(set(reduce(\
lambda x, y: list(x)+list(y),\
train_lex+valid_lex+test_lex)))
nclasses = len(set(reduce(\
lambda x, y: list(x)+list(y),\
train_y+test_y+valid_y)))
nsentences = len(train_lex)
# instantiate the model
numpy.random.seed(s['seed'])
random.seed(s['seed'])
rnn = model( nh = s['nhidden'],
nc = nclasses,
ne = vocsize,
de = s['emb_dimension'],
cs = s['win'],
depth = s['depth'] )
# train with early stopping on validation set
best_f1 = -numpy.inf
s['clr'] = s['lr']
for e in xrange(s['nepochs']):
# shuffle
shuffle([train_lex, train_ne, train_y], s['seed'])
s['ce'] = e
tic = time.time()
for i in xrange(nsentences):
cwords = contextwin(train_lex[i], s['win'])
words = map(lambda x: numpy.asarray(x).astype('int32'),\
minibatch(cwords, s['bs']))
labels = train_y[i]
for word_batch, label_last_word in zip(words, labels):
print word_batch
#print label_last_word
#pdb.set_trace()
rnn.train(word_batch, label_last_word, s['clr'])
rnn.normalize()
if s['verbose']:
print '[learning] epoch %i >> %2.2f%%'%(e,(i+1)*100./nsentences),'completed in %.2f (sec) <<\r'%(time.time()-tic),
sys.stdout.flush()
# evaluation // back into the real world : idx -> words
predictions_test = [ map(lambda x: idx2label[x], \
rnn.classify(numpy.asarray(contextwin(x, s['win'])).astype('int32')))\
for x in test_lex ]
groundtruth_test = [ map(lambda x: idx2label[x], y) for y in test_y ]
words_test = [ map(lambda x: idx2word[x], w) for w in test_lex]
predictions_valid = [ map(lambda x: idx2label[x], \
rnn.classify(numpy.asarray(contextwin(x, s['win'])).astype('int32')))\
for x in valid_lex ]
groundtruth_valid = [ map(lambda x: idx2label[x], y) for y in valid_y ]
words_valid = [ map(lambda x: idx2word[x], w) for w in valid_lex]
# evaluation // compute the accuracy using conlleval.pl
res_test = conlleval(predictions_test, groundtruth_test, words_test, folder + '/current.test.txt')
res_valid = conlleval(predictions_valid, groundtruth_valid, words_valid, folder + '/current.valid.txt')
if res_valid['f1'] > best_f1:
rnn.save(folder)
best_f1 = res_valid['f1']
if s['verbose']:
print 'NEW BEST: epoch', e, 'valid F1', res_valid['f1'], 'best test F1', res_test['f1'], ' '*20
s['vf1'], s['vp'], s['vr'] = res_valid['f1'], res_valid['p'], res_valid['r']
s['tf1'], s['tp'], s['tr'] = res_test['f1'], res_test['p'], res_test['r']
s['be'] = e
subprocess.call(['mv', folder + '/current.test.txt', folder + '/best.test.txt'])
subprocess.call(['mv', folder + '/current.valid.txt', folder + '/best.valid.txt'])
else:
print ''
# learning rate decay if no improvement in 10 epochs
if s['decay'] and abs(s['be']-s['ce']) >= 10: s['clr'] *= 0.5
if s['clr'] < 1e-5: break
print 'BEST RESULT: epoch', e, 'valid F1', s['vf1'], 'best test F1', s['tf1'], 'with the model', folder
def run(s) :
print s
folder = os.path.basename(__file__).split('.')[0]
if not os.path.exists(folder): os.mkdir(folder)
#print folder
# load the dataset
eval_options = []
if s['dataset'] == 'atis':
train_set, valid_set, test_set, dic = load.atisfold(s['fold'])
if s['dataset'] == 'ner':
train_set, valid_set, test_set, dic = load.ner()
if s['dataset'] == 'chunk':
train_set, valid_set, test_set, dic = load.chunk()
if s['dataset'] == 'pos':
train_set, valid_set, test_set, dic = load.pos()
eval_options = ['-r']
idx2label = dict((k, v) for v, k in dic['labels2idx'].iteritems())
idx2word = dict((k, v) for v, k in dic['words2idx'].iteritems())
train_lex, train_ne, train_y = train_set
valid_lex, valid_ne, valid_y = valid_set
test_lex, test_ne, test_y = test_set
vocsize = len(dic['words2idx'])
nclasses = len(dic['labels2idx'])
nsentences = len(train_lex)
wv = None
if 'WVFolderName' in s:
# load word vector
# wv = numpy.zeros((vocsize+1, s['emb_dimension']))
# input = open(s['wv_folder'] + str(s['emb_dimension']), 'r')
# for line in input:
# tokens = line.split(' ')
# wv[int(tokens[0])] = [float(tokens[j]) for j in xrange(1, len(tokens) - 1)]
# load word vector
wvnp = np.load("./../WV/" + s['WVFolderName'] + "/" + s['model']+".words" + str(s['emb_dimension']) + ".npy")
# load vocab
with open("./../WV/" + s['WVFolderName'] + "/" + s['model']+".words" + str(s['emb_dimension']) + ".vocab") as f:
vocab = [line.strip() for line in f if len(line) > 0]
wi = dict([(a, i) for i, a in enumerate(vocab)])
iw = vocab
wv = numpy.zeros((vocsize + 1, s['emb_dimension']))
random_v = math.sqrt(6.0 / numpy.sum(s['emb_dimension'])) * numpy.random.uniform(-1.0, 1.0, (s['emb_dimension']))
miss = 0
for i in range(0, vocsize):
word = idx2word[i]
if word in wi:
wv[i] = wvnp[wi[word]]
# print wvnp[wi[word]]
else:
wv[i] = random_v
miss += 1
print miss, '/', vocsize
best_valid = numpy.zeros(len(s['rho'])) - numpy.inf
best_test = numpy.zeros(len(s['rho'])) - numpy.inf
test_f1List = [[],[],[],[],[],[] ]
# print 111
# print test_f1List
# instanciate the model
numpy.random.seed(s['seed'])
random.seed(s['seed'])
rnn = elman_attention.model(nh=s['nhidden'],
nc=nclasses,
ne=vocsize,
de=s['emb_dimension'],
attention=s['attention'],
h_win=s['h_win'],
lvrg=s['lvrg'],
wv=wv)
# train with early stopping on validation set
for e in xrange(s['nepochs']):
# shuffle
shuffle([train_lex, train_ne, train_y], s['seed'])
s['ce'] = e
tic = time.time()
for i in xrange(nsentences):
cwords = contextwin(train_lex[i])
labels = train_y[i]
# for j in xrange(len(words)):
# if j >= 2 :
# rnn.train(words[j], [labels[j-2], labels[j-1], labels[j]], s['clr'])
nl, aaL = rnn.train(cwords, labels, s['dropRate'], 1)
#if i % 1 == 0:
# print aaL
# rnn.normalize()
if s['verbose']:
sys.stdout.write(('\r[learning] epoch %i >> %2.2f%%' % (
e, (i + 1) * 100. / nsentences) +
(' average speed in %.2f (min) <<' % (
(time.time() - tic) / 60 / (i + 1) * nsentences)) + (' completed in %.2f (sec) <<' % (
(time.time() - tic)))))
sys.stdout.flush()
print 'start test', time.time() / 60
# print avgSentenceLength / (nsentences)
# evaluation // back into the real world : idx -> words
# evaluation // back into the real world : idx -> words
print 'start pred train', time.time() / 60
predictions_train = [[map(lambda varible: idx2label[varible], w)\
for w in rnn.classify(numpy.asarray(contextwin(x)).astype('int32'), s['dropRate'], 0, s['rho'])]
for x in train_lex]
groundtruth_train = [map(lambda x: idx2label[x], y) for y in train_y]
words_train = [map(lambda x: idx2word[x], w) for w in train_lex]
#print 'start pred test', time.time() / 60
predictions_test = [[map(lambda varible: idx2label[varible], w)\
for w in rnn.classify(numpy.asarray(contextwin(x)).astype('int32'), s['dropRate'], 0, s['rho'])]
for x in test_lex]
groundtruth_test = [map(lambda x: idx2label[x], y) for y in test_y]
words_test = [map(lambda x: idx2word[x], w) for w in test_lex]
#print 'start pred valid', time.time() / 60
predictions_valid = [[map(lambda varible: idx2label[varible], w)\
for w in rnn.classify(numpy.asarray(contextwin(x)).astype('int32'), s['dropRate'], 0, s['rho'])]
for x in valid_lex]
groundtruth_valid = [map(lambda x: idx2label[x], y) for y in valid_y]
words_valid = [map(lambda x: idx2word[x], w) for w in valid_lex]
#print 'end pred, start eval', time.time() / 60
# evaluation // compute the accuracy using conlleval.pl
for i_rho in xrange(len(s['rho'])) :
ptrain = [p[i_rho] for p in predictions_train]
ptest = [p[i_rho] for p in predictions_test]
pvalid = [p[i_rho] for p in predictions_valid]
res_train = conlleval(ptrain, groundtruth_train, words_train, folder + '/current.train.txt' + str(s['seed']), eval_options)
res_test = conlleval(ptest, groundtruth_test, words_test, folder + '/current.test.txt' + str(s['seed']), eval_options)
res_valid = conlleval(pvalid, groundtruth_valid, words_valid, folder + '/current.valid.txt' + str(s['seed']), eval_options)
print ' epoch', e, ' rho ', i_rho, ' train p', res_train[
'p'], 'valid p', res_valid[
'p'],' train r', res_train[
'r'], 'valid r', res_valid[
'r'],' train F1', res_train[
'f1'], 'valid F1', res_valid[
'f1'], 'best test F1', res_test['f1'], ' ' * 20
test_f1List[i_rho].append(res_test['f1'])
if res_valid['f1'] > best_valid[i_rho]:
best_valid[i_rho] = res_valid['f1']
best_test[i_rho] = res_test['f1']
for i_rho in xrange(len(s['rho'])) :
print i_rho, s['dataset'],
if s['model'] == 'glove':
print s['WVFolderName'].replace('skip', 'glove'),
else:
print s['WVFolderName'],
for iff1 in test_f1List[i_rho]:
print iff1,
print ''
for i_rho in xrange(len(s['rho'])) :
print s['rho'][i_rho], ' ', best_valid[i_rho] , '/' , best_test[i_rho]
#print 'end eval', time.time() / 60
print 'BEST RESULT: epoch', e, 'valid F1', s['vf1'], 'best test F1', s['tf1'], 'with the model', folder
