def train(self, x, y, window_size, learning_rate):
cwords = contextwin(x, window_size)
words = list(map(lambda x: numpy.asarray(x).astype('int32'), cwords))
labels = y
self.sentence_train(words, labels, learning_rate)
if self.normal:
self.normalize()
def train(self, x, y, window_size, learning_rate):
cwords = contextwin(x, window_size)
words = list(map(lambda x: numpy.asarray(x).astype("int32"), cwords))
labels = y
self.sequence_train(words, labels, learning_rate)
if self.normal:
self.normalize()
def train(self, x, y, window_size, learning_rate):
cwords = contextwin(x, window_size)
words = list(map(lambda x: numpy.asarray(x).astype('int32'), cwords))
labels = y
# words is a list of arrays with same length
# [array([ -1, -1, -1, 276, 32, 481, 194]),
# array([ -1, -1, 276, 32, 481, 194, 480]),
# ...]
# it is later passed as idxs
# idxs is a matrix
# the transition is achieve somehow in theano, ex.
# func_test_f = 2*idxs
# func_test = theano.function(inputs=[idxs],outputs=func_test_f)
# func_test(words)
# array([[ -2, -2, -2, 552, 64, 962, 388],
# [ -2, -2, 552, 64, 962, 388, 960],
self.sentence_train(words, labels, learning_rate)
if self.normal:
self.normalize()
def test_rnnslu_embeddings(**kwargs):
"""
Wrapper function for training and testing RNNSLU
:type fold: int
:param fold: fold index of the ATIS dataset, from 0 to 4.
:type lr: float
:param lr: learning rate used (factor for the stochastic gradient.
:type nepochs: int
:param nepochs: maximal number of epochs to run the optimizer.
:type win: int
:param win: number of words in the context window.
:type nhidden: int
:param n_hidden: number of hidden units.
:type emb_dimension: int
:param emb_dimension: dimension of word embedding.
:type verbose: boolean
:param verbose: to print out epoch summary or not to.
:type decay: boolean
:param decay: decay on the learning rate if improvement stop.
:type savemodel: boolean
:param savemodel: save the trained model or not.
:type normal: boolean
:param normal: normalize word embeddings after each update or not.
:type folder: string
:param folder: path to the folder where results will be stored.
"""
# process input arguments
param = {
"fold": 3,
"lr": 0.0970806646812754,
"verbose": True,
"decay": True,
"win": 7,
"nhidden": 200,
"seed": 345,
"emb_dimension": 50,
"nepochs": 60,
"savemodel": False,
"normal": True,
"folder": "../result",
}
param_diff = set(kwargs.keys()) - set(param.keys())
if param_diff:
raise KeyError("invalid arguments:" + str(tuple(param_diff)))
param.update(kwargs)
if param["verbose"]:
for k, v in param.items():
print("%s: %s" % (k, v))
# create result folder if not exists
check_dir(param["folder"])
# load the dataset
print("... loading the dataset")
train_set, valid_set, test_set, dic = load_data(param["fold"])
# create mapping from index to label, and index to word
idx2label = dict((k, v) for v, k in dic["labels2idx"].items())
idx2word = dict((k, v) for v, k in dic["words2idx"].items())
# unpack dataset
train_lex, train_ne, train_y = train_set
valid_lex, valid_ne, valid_y = valid_set
test_lex, test_ne, test_y = test_set
vocsize = len(dic["words2idx"])
nclasses = len(dic["labels2idx"])
nsequences = len(train_lex)
groundtruth_valid = [map(lambda x: idx2label[x], y) for y in valid_y]
words_valid = [map(lambda x: idx2word[x], w) for w in valid_lex]
groundtruth_test = [map(lambda x: idx2label[x], y) for y in test_y]
words_test = [map(lambda x: idx2word[x], w) for w in test_lex]
# instanciate the model
numpy.random.seed(param["seed"])
random.seed(param["seed"])
print("... building the model")
rnn = RNNSLU(nh=param["nhidden"], nc=nclasses, ne=vocsize, de=param["emb_dimension"], cs=param["win"])
# train with early stopping on validation set
print("... training")
best_f1 = -numpy.inf
param["clr"] = param["lr"]
for e in range(param["nepochs"]):
# shuffle
shuffle([train_lex, train_ne, train_y], param["seed"])
param["ce"] = e
tic = timeit.default_timer()
for i, (x, y) in enumerate(zip(train_lex, train_y)):
rnn.train(x, y, param["win"], param["clr"])
print("[learning] epoch %i >> %2.2f%%" % (e, (i + 1) * 100.0 / nsequences), end=" ")
print("completed in %.2f (sec) <<\r" % (timeit.default_timer() - tic), end="")
sys.stdout.flush()
# evaluation // back into the real world : idx -> words
predictions_test = [
map(lambda x: idx2label[x], rnn.classify(numpy.asarray(contextwin(x, param["win"])).astype("int32")))
for x in test_lex
]
predictions_valid = [
map(lambda x: idx2label[x], rnn.classify(numpy.asarray(contextwin(x, param["win"])).astype("int32")))
for x in valid_lex
]
# evaluation // compute the accuracy using conlleval.pl
res_test = conlleval(
predictions_test, groundtruth_test, words_test, param["folder"] + "/current.test.txt", param["folder"]
)
res_valid = conlleval(
predictions_valid, groundtruth_valid, words_valid, param["folder"] + "/current.valid.txt", param["folder"]
)
if res_valid["f1"] > best_f1:
if param["savemodel"]:
rnn.save(param["folder"])
best_rnn = copy.deepcopy(rnn)
best_f1 = res_valid["f1"]
if param["verbose"]:
print("NEW BEST: epoch", e, "valid F1", res_valid["f1"], "best test F1", res_test["f1"])
param["vf1"], param["tf1"] = res_valid["f1"], res_test["f1"]
param["vp"], param["tp"] = res_valid["p"], res_test["p"]
param["vr"], param["tr"] = res_valid["r"], res_test["r"]
param["be"] = e
os.rename(param["folder"] + "/current.test.txt", param["folder"] + "/best.test.txt")
os.rename(param["folder"] + "/current.valid.txt", param["folder"] + "/best.valid.txt")
else:
if param["verbose"]:
print("")
# learning rate decay if no improvement in 10 epochs
if param["decay"] and abs(param["be"] - param["ce"]) >= 10:
param["clr"] *= 0.5
rnn = best_rnn
if param["clr"] < 1e-5:
break
print(
"BEST RESULT: epoch",
param["be"],
"valid F1",
param["vf1"],
"best test F1",
param["tf1"],
"with the model",
param["folder"],
)
return rnn, idx2word
def test_rnn_parity2(**kwargs):
param = {
"nbits": 3,
"nnums": [1000, 500, 100],
"lr": 0.1,
"verbose": True,
"decay": True,
"win": 7,
"nhidden": 200,
"seed": 420,
"nepochs": 60,
"savemodel": False,
"folder": "../result",
}
param_diff = set(kwargs.keys()) - set(param.keys())
if param_diff:
raise KeyError("invalid arguments:" + str(tuple(param_diff)))
param.update(kwargs)
if param["verbose"]:
for k, v in param.items():
print("%s: %s" % (k, v))
# create result folder if not exists
check_dir(param["folder"])
# load the dataset
print("... loading the dataset")
# generate datasets
numpy.random.seed(param["seed"]) # Gaurantees consistency across runs
train_x, train_y = gen_parity_pair(param["nbits"], param["nnums"][0])
valid_x, valid_y = gen_parity_pair(param["nbits"], param["nnums"][1])
test_x, test_y = gen_parity_pair(param["nbits"], param["nnums"][2])
# Converting data into a sequence
train_y = CreateSeqYs(train_x)
valid_y = CreateSeqYs(valid_x)
test_y = CreateSeqYs(test_x)
# train_y = AddXsToY(train_x, train_y)
# train_x = AddZerosToX(train_x)
# valid_y = AddXsToY(valid_x, valid_y)
# valid_x = AddZerosToX(valid_x)
# test_y = AddXsToY(test_x, test_y)
# test_x = AddZerosToX(test_x)
# These are set to 2 because the classes are either 1 or 0 for the labels and features
nclasses = 2
vocsize = 2
nsequences = len(train_x)
# instanciate the model
random.seed(param["seed"])
print("... building the model")
rnn = myRNN2(nh=param["nhidden"], nc=nclasses, ne=vocsize, cs=param["win"])
# train with early stopping on validation set
print("... training")
best_f1 = -numpy.inf
param["clr"] = param["lr"]
extract_col = train_x.shape[1] - 1
for e in range(param["nepochs"]):
param["ce"] = e
tic = timeit.default_timer()
for i, (x, y) in enumerate(zip(train_x, train_y)):
rnn.train(numpy.asarray(x), numpy.asarray(y), param["win"], param["clr"])
# rnn.train(x, y, param['win'], param['clr'])
# print('[learning] epoch %i >> %2.2f%%' % ( e, (i + 1) * 100. / nsequences))
# print('completed in %.2f (sec) <<\r' % (timeit.default_timer() - tic))
sys.stdout.flush()
# evaluation // back into the real world
predictions_train = numpy.asarray(
[rnn.classify(numpy.asarray(contextwin(x, param["win"])).astype("int32")) for x in train_x]
)
predictions_valid = numpy.asarray(
[rnn.classify(numpy.asarray(contextwin(x, param["win"])).astype("int32")) for x in valid_x]
)
predictions_test = numpy.asarray(
[rnn.classify(numpy.asarray(contextwin(x, param["win"])).astype("int32")) for x in test_x]
)
# E Computing the f1
res_train = metrics.f1_score(train_y[:, extract_col], predictions_train[:, extract_col])
res_valid = metrics.f1_score(valid_y[:, extract_col], predictions_valid[:, extract_col])
res_test = metrics.f1_score(test_y[:, extract_col], predictions_test[:, extract_col])
if res_valid > best_f1:
best_rnn = copy.deepcopy(rnn)
best_f1 = res_valid
if param["verbose"]:
print("NEW BEST: epoch", e, "training F1", res_train, "valid F1", res_valid, "best test F1", res_test)
param["vf1"], param["tf1"] = res_valid, res_test
param["be"] = e
else:
if param["verbose"]:
print("")
# learning rate decay if no improvement in 10 epochs
if param["decay"] and abs(param["be"] - param["ce"]) >= 10:
param["clr"] *= 0.5
rnn = best_rnn
if param["clr"] < 1e-5:
break
print(
"BEST RESULT: epoch",
param["be"],
"valid F1",
param["vf1"],
"best test F1",
param["tf1"],
"with the model",
param["folder"],
)
def test_rnnslu_n(**kwargs):
"""
Wrapper function for training and testing RNNSLU
:type fold: int
:param fold: fold index of the ATIS dataset, from 0 to 4.
:type lr: float
:param lr: learning rate used (factor for the stochastic gradient.
:type nepochs: int
:param nepochs: maximal number of epochs to run the optimizer.
:type win: int
:param win: number of words in the context window.
:type nhidden: int
:param n_hidden: number of hidden units.
:type emb_dimension: int
:param emb_dimension: dimension of word embedding.
:type verbose: boolean
:param verbose: to print out epoch summary or not to.
:type decay: boolean
:param decay: decay on the learning rate if improvement stop.
:type savemodel: boolean
:param savemodel: save the trained model or not.
:type normal: boolean
:param normal: normalize word embeddings after each update or not.
:type folder: string
:param folder: path to the folder where results will be stored.
"""
# process input arguments
param = {
'fold': 3,
'lr': 0.0970806646812754,
'verbose': True,
'decay': True,
'win': 7,
'nhidden': 200,
'seed': 345,
'emb_dimension': 50,
'nepochs': 60,
'savemodel': False,
'normal': True,
'folder': '../result'
}
param_diff = set(kwargs.keys()) - set(param.keys())
if param_diff:
raise KeyError("invalid arguments:" + str(tuple(param_diff)))
param.update(kwargs)
if param['verbose']:
for k, v in param.items():
print("%s: %s" % (k, v))
# create result folder if not exists
check_dir(param['folder'])
# load the dataset
print('... loading the dataset')
train_set, valid_set, test_set, dic = load_data(param['fold'])
# create mapping from index to label, and index to word
idx2label = dict((k, v) for v, k in dic['labels2idx'].items())
idx2word = dict((k, v) for v, k in dic['words2idx'].items())
# unpack dataset
train_lex, train_ne, train_y = train_set
valid_lex, valid_ne, valid_y = valid_set
test_lex, test_ne, test_y = test_set
vocsize = len(dic['words2idx'])
nclasses = len(dic['labels2idx'])
nsentences = len(train_lex)
groundtruth_valid = [map(lambda x: idx2label[x], y) for y in valid_y]
words_valid = [map(lambda x: idx2word[x], w) for w in valid_lex]
groundtruth_test = [map(lambda x: idx2label[x], y) for y in test_y]
words_test = [map(lambda x: idx2word[x], w) for w in test_lex]
# instanciate the model
numpy.random.seed(param['seed'])
random.seed(param['seed'])
print('... building the model')
rnn = RNNSLU_n(nh=param['nhidden'],
nc=nclasses,
ne=vocsize,
de=param['emb_dimension'],
cs=param['win'],
normal=param['normal'])
# train with early stopping on validation set
print('... training')
best_f1 = -numpy.inf
param['clr'] = param['lr']
for e in range(param['nepochs']):
# shuffle
shuffle_rnn([train_lex, train_ne, train_y], param['seed'])
param['ce'] = e
tic = timeit.default_timer()
for i, (x, y) in enumerate(zip(train_lex, train_y)):
rnn.train(x, y, param['win'], param['clr'])
print('[learning] epoch %i >> %2.2f%%' %
(e, (i + 1) * 100. / nsentences),
end=' ')
print('completed in %.2f (sec) <<\r' %
(timeit.default_timer() - tic),
end='')
sys.stdout.flush()
# evaluation // back into the real world : idx -> words
predictions_test = [
map(
lambda x: idx2label[x],
rnn.classify(
numpy.asarray(contextwin(x,
param['win'])).astype('int32')))
for x in test_lex
]
predictions_valid = [
map(
lambda x: idx2label[x],
rnn.classify(
numpy.asarray(contextwin(x,
param['win'])).astype('int32')))
for x in valid_lex
]
# evaluation // compute the accuracy using conlleval.pl
res_test = conlleval(predictions_test, groundtruth_test, words_test,
param['folder'] + '/current.test.txt',
param['folder'])
res_valid = conlleval(predictions_valid, groundtruth_valid,
words_valid,
param['folder'] + '/current.valid.txt',
param['folder'])
if res_valid['f1'] > best_f1:
if param['savemodel']:
rnn.save(param['folder'])
best_rnn = copy.deepcopy(rnn)
best_f1 = res_valid['f1']
if param['verbose']:
print('NEW BEST: epoch', e, 'valid F1', res_valid['f1'],
'best test F1', res_test['f1'])
param['vf1'], param['tf1'] = res_valid['f1'], res_test['f1']
param['vp'], param['tp'] = res_valid['p'], res_test['p']
param['vr'], param['tr'] = res_valid['r'], res_test['r']
param['be'] = e
os.rename(param['folder'] + '/current.test.txt',
param['folder'] + '/best.test.txt')
os.rename(param['folder'] + '/current.valid.txt',
param['folder'] + '/best.valid.txt')
else:
if param['verbose']:
print('')
# learning rate decay if no improvement in 10 epochs
if param['decay'] and abs(param['be'] - param['ce']) >= 10:
param['clr'] *= 0.5
rnn = best_rnn
if param['clr'] < 1e-5:
break
print('BEST RESULT: epoch', param['be'], 'valid F1', param['vf1'],
'best test F1', param['tf1'], 'with the model', param['folder'])
return rnn, dic
def test_rnnslu2(kwargs):
"""
Wrapper function for training and testing RNNSLU
:type fold: int
:param fold: fold index of the ATIS dataset, from 0 to 4.
:type lr: float
:param lr: learning rate used (factor for the stochastic gradient.
:type nepochs: int
:param nepochs: maximal number of epochs to run the optimizer.
:type win: int
:param win: number of words in the context window.
:type nhidden1: int
:param n_hidden: number of hidden units in the first hidden layer.
:type nhidden2: int
:param n_hidden: number of hidden units in the second hidden layer.
:type emb_dimension: int
:param emb_dimension: dimension of word embedding.
:type verbose: boolean
:param verbose: to print out epoch summary or not to.
:type decay: boolean
:param decay: decay on the learning rate if improvement stop.
:type savemodel: boolean
:param savemodel: save the trained model or not.
:type folder: string
:param folder: path to the folder where results will be stored.
"""
# process input arguments
param = {
'fold': 3,
'lr': 0.0970806646812754,
'verbose': True,
'decay': True,
'win': 7,
'nhidden1': 200,
'nhidden2': 200,
'seed': 345,
'emb_dimension': 50,
'nepochs': 60,
'savemodel': False,
'normal': True,
'folder':'../result'}
param_diff = set(kwargs.keys()) - set(param.keys())
if param_diff:
raise KeyError("invalid arguments:" + str(tuple(param_diff)))
param.update(kwargs)
if param['verbose']:
for k,v in param.items():
print("%s: %s" % (k,v))
# create result folder if not exists
check_dir(param['folder'])
# load the dataset
print('... loading the dataset')
train_set, valid_set, test_set, dic = load_data(param['fold'])
# create mapping from index to label, and index to word
idx2label = dict((k, v) for v, k in dic['labels2idx'].items())
idx2word = dict((k, v) for v, k in dic['words2idx'].items())
# unpack dataset
train_lex, train_ne, train_y = train_set
valid_lex, valid_ne, valid_y = valid_set
test_lex, test_ne, test_y = test_set
vocsize = len(dic['words2idx'])
nclasses = len(dic['labels2idx'])
nsentences = len(train_lex)
groundtruth_valid = [map(lambda x: idx2label[x], y) for y in valid_y]
words_valid = [map(lambda x: idx2word[x], w) for w in valid_lex]
groundtruth_test = [map(lambda x: idx2label[x], y) for y in test_y]
words_test = [map(lambda x: idx2word[x], w) for w in test_lex]
# instanciate the model
numpy.random.seed(param['seed'])
random.seed(param['seed'])
# TODO
print('... building the model')
rnn = RNNSLU2(nh=param['nhidden1'],
nh2=param['nhidden2'],
nc=nclasses,
ne=vocsize,
de=param['emb_dimension'],
cs=param['win'],
normal=param['normal'])
# train with early stopping on validation set
print('... training')
best_f1 = -numpy.inf
param['clr'] = param['lr']
for e in range(param['nepochs']):
# shuffle
shuffle([train_lex, train_ne, train_y], param['seed'])
param['ce'] = e
tic = timeit.default_timer()
for i, (x, y) in enumerate(zip(train_lex, train_y)):
rnn.train(x, y, param['win'], param['clr'])
print('[learning] epoch %i >> %2.2f%%' % (
e, (i + 1) * 100. / nsentences), end=' ')
print('completed in %.2f (sec) <<\r' % (timeit.default_timer() - tic), end='')
sys.stdout.flush()
# evaluation // back into the real world : idx -> words
predictions_test = [map(lambda x: idx2label[x],
rnn.classify(numpy.asarray(
contextwin(x, param['win'])).astype('int32')))
for x in test_lex]
predictions_valid = [map(lambda x: idx2label[x],
rnn.classify(numpy.asarray(
contextwin(x, param['win'])).astype('int32')))
for x in valid_lex]
# evaluation // compute the accuracy using conlleval.pl
res_test = conlleval(predictions_test,
groundtruth_test,
words_test,
param['folder'] + '/current.test.txt',
param['folder'])
res_valid = conlleval(predictions_valid,
groundtruth_valid,
words_valid,
param['folder'] + '/current.valid.txt',
param['folder'])
if res_valid['f1'] > best_f1:
if param['savemodel']:
rnn.save(param['folder'])
best_rnn = copy.deepcopy(rnn)
best_f1 = res_valid['f1']
if param['verbose']:
print('NEW BEST: epoch', e,
'valid F1', res_valid['f1'],
'best test F1', res_test['f1'])
param['vf1'], param['tf1'] = res_valid['f1'], res_test['f1']
param['vp'], param['tp'] = res_valid['p'], res_test['p']
param['vr'], param['tr'] = res_valid['r'], res_test['r']
param['be'] = e
os.rename(param['folder'] + '/current.test.txt',
param['folder'] + '/best.test.txt')
os.rename(param['folder'] + '/current.valid.txt',
param['folder'] + '/best.valid.txt')
else:
if param['verbose']:
print('')
# learning rate decay if no improvement in 10 epochs
if param['decay'] and abs(param['be']-param['ce']) >= 10:
param['clr'] *= 0.5
rnn = best_rnn
if param['clr'] < 1e-5:
break
print('BEST RESULT: epoch', param['be'],
'valid F1', param['vf1'],
'best test F1', param['tf1'],
'with the model', param['folder'])
def test_lstm_parity(n_bit=8,
nh=200,
de=50,
n_epoch=1000,
learning_rate=0.1,
decay_period=10):
# generate datasets
train_set = gen_parity_pair(n_bit, 1000)
valid_set = gen_parity_pair(n_bit, 500)
test_set = gen_parity_pair(n_bit, 100)
train_set_x = train_set[0]
train_set_y = train_set[1]
valid_set_x = valid_set[0]
valid_set_y = valid_set[1]
test_set_x = test_set[0]
test_set_y = test_set[1]
''' T
# Convert raw dataset to Theano shared variables.
train_set_x, train_set_y = shared_dataset(train_set)
valid_set_x, valid_set_y = shared_dataset(valid_set)
test_set_x, test_set_y = shared_dataset(test_set)
'''
nc = 2
ne = 2
cs = 1
# instanciate the model
seed = random.randint(0, 4294967295)
numpy.random.seed(seed)
random.seed(seed)
print('... building the model')
rnn = LSTM(nh=nh, nc=nc, ne=ne, de=de, cs=cs, normal=False)
# train with early stopping on validation set
print('... training')
best_f1 = -numpy.inf
decay = True
verbose = True
decay_cnt = 0
for e in range(n_epoch):
print("========== epoch %i ==========") % (e)
print("===== learning_rate: %f =====") % (learning_rate)
'''
# shuffle
seed = random.randint(0, 4294967295)
shuffle_rnn(train_set_x, seed)
'''
current_epoch = e
tic = timeit.default_timer()
for i, (x, y) in enumerate(zip(train_set_x, train_set_y)):
error = rnn.train(x, y, cs, learning_rate)
print(error)
'''
############################################################33
A = rnn.get_param()
print("A")
print("self.emb")
print(type(A[0].get_value()))
print(A[0].get_value())
print("self.wx")
print(type(A[1].get_value()))
print(A[1].get_value())
print("self.wh")
print(type(A[2].get_value()))
print(A[2].get_value())
print("self.w")
print(type(A[3].get_value()))
print(A[3].get_value())
print("self.h0")
print(type(A[4].get_value()))
print(A[4].get_value())
raw_input()
'''
print('completed in %.2f (sec) <<\r' % (timeit.default_timer() - tic))
sys.stdout.flush()
# evaluation // back into the real world : idx -> words
predictions_test = [
rnn.classify(numpy.asarray(contextwin(x, cs)).astype('int32'))
for x in test_set_x
] ################T .get_value()
predictions_valid = [
rnn.classify(numpy.asarray(contextwin(x, cs)).astype('int32'))
for x in valid_set_x
] ###############T .get_value()
'''
for x in test_set_x.get_value():
print("x")
print(type(x))
print(x)
print("numpy.asarray(contextwin(x, cs)).astype('int32')")
print(type(numpy.asarray(contextwin(x, cs)).astype('int32')))
print(numpy.asarray(contextwin(x, cs)).astype('int32'))
print("rnn.classify(numpy.asarray(contextwin(x, cs)).astype('int32'))")
print(type(rnn.classify(numpy.asarray(contextwin(x, cs)).astype('int32'))))
print(rnn.classify(numpy.asarray(contextwin(x, cs)).astype('int32')))
raw_input()
print("predictions_test")
print(type(predictions_test))
print(predictions_test)
print
print("test_set_y.eval()")
print(type(test_set_y.eval()))
print(test_set_y.eval())
'''
# evaluation // compute the accuracy using conlleval.pl
res_test = 0.0
for i in range(len(predictions_test)):
if (predictions_test[i][n_bit -
1] == test_set_y[i][n_bit -
1]): #T .eval()
res_test += 1.0
res_test /= float(len(predictions_test))
res_valid = 0.0
for i in range(len(predictions_valid)):
if (predictions_valid[i][n_bit -
1] == valid_set_y[i][n_bit -
1]): #T .eval()
res_valid += 1.0
res_valid /= float(len(predictions_valid))
#############################################3
print("res_valid")
print(res_valid)
print("res_test")
print(res_test)
'''
print("predictions_test")
print(predictions_test)
print("test_set_y.eval()")
print(test_set_y.eval())
'''
if res_valid > best_f1:
best_rnn = copy.deepcopy(rnn)
best_f1 = res_valid
best_tf1 = res_test
best_epoch = e
if verbose:
print('NEW BEST: epoch', e, 'valid F1', res_valid,
'best test F1', res_test)
if res_test == 1.0:
break
if decay == False:
decay_cnt += 1
if (decay_cnt > 8):
decay = True
# learning rate decay if no improvement in "decay_period" epochs
if decay and abs(best_epoch - current_epoch) >= decay_period:
learning_rate *= 0.5
rnn = best_rnn
decay = False
decay_cnt = 0
if learning_rate < 1e-5:
break
print('BEST RESULT: epoch', best_epoch, 'valid F1', best_f1,
'best test F1', best_tf1)
return rnn
def test_lstm(**kwargs):
# process input arguments
param = {
'fold': 3,
'lr': 0.1,
'verbose': True,
'decay': True,
'win': 3,
'nhidden1': 300,
'nhidden2': 0,
'seed': 345,
'emb_dimension': 50,
'nepochs': 20,
'savemodel': False,
'folder': '../result'
}
param_diff = set(kwargs.keys()) - set(param.keys())
if param_diff:
raise KeyError("invalid arguments:" + str(tuple(param_diff)))
param.update(kwargs)
if param['verbose']:
for k, v in param.items():
print("%s: %s" % (k, v))
# create result folder if not exists
check_dir(param['folder'])
# load the dataset
print('... loading the dataset')
train_set, valid_set, test_set, dic = load_data(param['fold'])
# create mapping from index to label, and index to word
idx2label = dict((k, v) for v, k in dic['labels2idx'].items())
idx2word = dict((k, v) for v, k in dic['words2idx'].items())
train_lex_temp, train_ne, train_y_temp = train_set
valid_lex_temp, valid_ne, valid_y_temp = valid_set
test_lex_temp, test_ne, test_y_temp = test_set
train_lex = (7) * numpy.ones((len(train_lex_temp), 30))
for i in range(len(train_lex_temp)):
for j in range(min(30, len(train_lex_temp[i]))):
train_lex[i][j] = train_lex_temp[i][j]
test_lex = (7) * numpy.ones((len(test_lex_temp), 30))
for i in range(len(test_lex_temp)):
for j in range(min(30, len(test_lex_temp[i]))):
test_lex[i][j] = test_lex_temp[i][j]
valid_lex = (7) * numpy.ones((len(valid_lex_temp), 30))
for i in range(len(valid_lex_temp)):
for j in range(min(30, len(valid_lex_temp[i]))):
valid_lex[i][j] = valid_lex_temp[i][j]
train_y = 126 * numpy.ones((len(train_y_temp), 30))
for i in range(len(train_y_temp)):
for j in range(min(30, len(train_y_temp[i]))):
train_y[i][j] = train_y_temp[i][j]
test_y = 126 * numpy.ones((len(test_y_temp), 30))
for i in range(len(test_y_temp)):
for j in range(min(30, len(test_y_temp[i]))):
test_y[i][j] = test_y_temp[i][j]
valid_y = 126 * numpy.ones((len(valid_y_temp), 30))
for i in range(len(valid_y_temp)):
for j in range(min(30, len(valid_y_temp[i]))):
valid_y[i][j] = valid_y_temp[i][j]
vocsize = len(dic['words2idx'])
nclasses = len(dic['labels2idx'])
nsentences = len(train_lex)
groundtruth_valid = [map(lambda x: idx2label[x], y) for y in valid_y]
words_valid = [map(lambda x: idx2word[x], w) for w in valid_lex]
groundtruth_test = [map(lambda x: idx2label[x], y) for y in test_y]
words_test = [map(lambda x: idx2word[x], w) for w in test_lex]
# instanciate the model
numpy.random.seed(param['seed'])
random.seed(param['seed'])
# TODO
print('... building the model')
rnn = LSTM(nh=param['nhidden1'],
nh2=param['nhidden2'],
nc=nclasses,
ne=vocsize,
de=param['emb_dimension'],
cs=param['win'])
# train with early stopping on validation set
print('... training')
best_f1 = -numpy.inf
param['clr'] = param['lr']
for e in range(param['nepochs']):
# shuffle
shuffle([train_lex, train_ne, train_y], param['seed'])
param['ce'] = e
tic = timeit.default_timer()
##########################
#print(rnn.showh2(numpy.asarray(contextwin(test_lex[0], param['win'])).astype('int32')))
#########################
for i, (x, y) in enumerate(zip(train_lex, train_y)):
rnn.train(x, y, param['win'], param['clr'])
print('[learning] epoch %i >> %2.2f%%' %
(e, (i + 1) * 100. / nsentences),
end=' ')
print('completed in %.2f (sec) <<\r' %
(timeit.default_timer() - tic),
end='')
sys.stdout.flush()
# evaluation // back into the real world : idx -> words
predictions_test = [
map(
lambda x: idx2label[x],
rnn.classify(
numpy.asarray(contextwin(x,
param['win'])).astype('int32')))
for x in test_lex
]
predictions_valid = [
map(
lambda x: idx2label[x],
rnn.classify(
numpy.asarray(contextwin(x,
param['win'])).astype('int32')))
for x in valid_lex
]
# evaluation // compute the accuracy using conlleval.pl
res_test = conlleval(predictions_test, groundtruth_test, words_test,
param['folder'] + '/current.test.txt',
param['folder'])
res_valid = conlleval(predictions_valid, groundtruth_valid,
words_valid,
param['folder'] + '/current.valid.txt',
param['folder'])
if res_valid['f1'] > best_f1:
if param['savemodel']:
rnn.save(param['folder'])
best_rnn = copy.deepcopy(rnn)
best_f1 = res_valid['f1']
if param['verbose']:
print('NEW BEST: epoch', e, 'valid F1', res_valid['f1'],
'best test F1', res_test['f1'])
param['vf1'], param['tf1'] = res_valid['f1'], res_test['f1']
param['vp'], param['tp'] = res_valid['p'], res_test['p']
param['vr'], param['tr'] = res_valid['r'], res_test['r']
param['be'] = e
os.rename(param['folder'] + '/current.test.txt',
param['folder'] + '/best.test.txt')
os.rename(param['folder'] + '/current.valid.txt',
param['folder'] + '/best.valid.txt')
else:
if param['verbose']:
print('')
# learning rate decay if no improvement in 10 epochs
if param['decay'] and abs(param['be'] - param['ce']) >= 10:
param['clr'] *= 0.5
rnn = best_rnn
if param['clr'] < 1e-5:
break
print('BEST RESULT: epoch', param['be'], 'valid F1', param['vf1'],
'best test F1', param['tf1'], 'with the model', param['folder'])
def test_rnn_parity(**kwargs):
"""
Wrapper function for training and testing RNNSLU
:type fold: int
:param fold: fold index of the ATIS dataset, from 0 to 4.
:type lr: float
:param lr: learning rate used (factor for the stochastic gradient.
:type nepochs: int
:param nepochs: maximal number of epochs to run the optimizer.
:type win: int
:param win: number of words in the context window.
:type nhidden: int
:param n_hidden: number of hidden units.
:type emb_dimension: int
:param emb_dimension: dimension of word embedding.
:type verbose: boolean
:param verbose: to print out epoch summary or not to.
:type decay: boolean
:param decay: decay on the learning rate if improvement stop.
:type savemodel: boolean
:param savemodel: save the trained model or not.
:type normal: boolean
:param normal: normalize word embeddings after each update or not.
:type folder: string
:param folder: path to the folder where results will be stored.
"""
param = {
'nbits':3,
'nsamples': [1000, 500, 100],
'lr': 0.1,
'verbose': True,
'decay': True,
'win': 7,
'nhidden': 200,
'nepochs': 60,
'savemodel': False,
'folder':'../result'
}
param_diff = set(kwargs.keys()) - set(param.keys())
if param_diff:
raise KeyError("invalid arguments:" + str(tuple(param_diff)))
param.update(kwargs)
if param['verbose']:
for k,v in param.items():
print("%s: %s" % (k,v))
# create result folder if not exists
check_dir(param['folder'])
# load the dataset
print('... loading the dataset')
# generate datasets
numpy.random.seed(100) # Gaurantees consistency across runs
train_x, train_y = gen_parity_pair(param['nbits'], param['nsamples'][0])
valid_x, valid_y = gen_parity_pair(param['nbits'], param['nsamples'][1])
test_x, test_y = gen_parity_pair(param['nbits'], param['nsamples'][2])
# need to re-create y to match X for training, test, and validation sets
train_y = makeY(train_x)
valid_y = makeY(valid_x)
test_y = makeY(test_x)
# Set nclass to 2 for 0 or 1
nclasses=2
vocsize = 2
nsequences = len(train_x)
random.seed(100)
print('... building the model')
rnn = RNN(
nh=param['nhidden'],
nc=nclasses,
ne=vocsize,
cs=param['win']
)
# train with early stopping on validation set
print('... training')
best_f1 = -numpy.inf
param['clr'] = param['lr']
extract_col = train_x.shape[1]-1
for e in range(param['nepochs']):
param['ce'] = e
tic = timeit.default_timer()
for i, (x, y) in enumerate(zip(train_x, train_y)):
rnn.train(numpy.asarray(x), numpy.asarray(y), param['win'], param['clr'])
sys.stdout.flush()
# evaluation and prediction
pred_train = numpy.asarray([rnn.classify(numpy.asarray( contextwin(x, param['win'])).astype('int32')) for x in train_x ])
pred_valid = numpy.asarray([rnn.classify(numpy.asarray( contextwin(x, param['win'])).astype('int32')) for x in valid_x ])
pred_test = numpy.asarray([rnn.classify(numpy.asarray( contextwin(x, param['win'])).astype('int32')) for x in test_x ])
# F1
res_train = metrics.f1_score(train_y[:, extract_col], pred_train[:, extract_col])
res_valid = metrics.f1_score(valid_y[:, extract_col], pred_valid[:, extract_col])
res_test = metrics.f1_score(test_y[:, extract_col], pred_test[:, extract_col])
print(numpy.sum(pred_test[:,extract_col]), numpy.sum(train_y[:,extract_col]))
if res_valid > best_f1:
best_rnn = copy.deepcopy(rnn)
best_f1 = res_valid
if param['verbose']:
print('NEW BEST: epoch', e,
'training F1', res_train,
'valid F1', res_valid,
'best test F1', res_test)
param['vf1'], param['tf1'] = res_valid, res_test
param['be'] = e
else:
if param['verbose']:
print('')
# learning rate decay if no improvement in 10 epochs
if param['decay'] and abs(param['be']-param['ce']) >= 10:
param['clr'] *= 0.5
rnn = best_rnn
if param['clr'] < 1e-5:
break
print('BEST RESULT: epoch', param['be'],
'valid F1', param['vf1'],
'best test F1', param['tf1'],
'with the model', param['folder'])
