def train(num_layers,num_hiddens,num_steps):
X_eval = None
y_eval = None
X_train, y_train,X_eval,y_eval= util.getTrainEvalData(train_path=train_path,
seq_max_len=num_steps)
word2id, _ = util.loadMap('data/word2id')
label2id, _ = util.loadMap('data/label2id')
num_words = len(word2id)
num_classes = len(label2id)
if emb_path != None:
embedding_matrix = util.getEmbedding(emb_path)
else:
embedding_matrix = None
with tf.Session() as sess:
initializer = tf.random_uniform_initializer(-0.2, 0.2)
with tf.variable_scope("model", reuse=None, initializer=initializer):
model = BILSTM(
num_words,
num_classes,
num_hiddens=num_hiddens,
num_steps=num_steps,
num_layers=num_layers,
embedding_matrix=embedding_matrix,
emb_dim=emb_dim)
# self,num_hiddens,num_steps,embedding_matrix,num_words,num_classes,is_training=True
init = tf.initialize_all_variables()
sess.run(init)
model.train(sess, model_path, X_train, y_train, X_eval, y_eval)
def test(num_layers,num_hiddens,num_steps):
word2id, _ = util.loadMap('data/word2id')
label2id, _ = util.loadMap('data/label2id')
num_words = len(word2id)
num_classes = len(label2id)
if emb_path != None:
embedding_matrix = util.getEmbedding(emb_path)
else:
embedding_matrix = None
X_test, X_test_str, X_test_label_str = util.getTestData(test_path, num_steps, False)
util.save_test_data(X_test_str, X_test_label_str, output_path=eval_path)
with tf.Session() as sess:
initializer = tf.random_uniform_initializer(-0.2, 0.2)
with tf.variable_scope("model", reuse=None, initializer=initializer):
model = BILSTM(
num_words,
num_classes,
num_hiddens=num_hiddens,
num_steps=num_steps,
num_layers=num_layers,
embedding_matrix=embedding_matrix,
emb_dim=emb_dim)
print 'loading lstm model parameters!'
saver = tf.train.Saver()
saver.restore(sess, model_path)
model.test(sess, X_test, X_test_str, output_path=output_path)
end_time = time.time()
evaluation.evalution(gold_path=eval_path, pred_path=output_path)
def __init__(self, worldname):
# A dictionary of tuple -> mapname
self.world = {}
# A dictionary of mapname -> TerrainLayer (initially, None)
self.terrain = {}
self.north = {}
self.south = {}
self.east = {}
self.west = {}
f = util.loadMap(worldname)
for line in f:
line = line.rstrip()
id, val = line.split()
if id == WorldLoader.ROWS:
self.rows = int(val)
elif id == WorldLoader.COLS:
self.cols = int(val)
elif id == WorldLoader.DIR:
self.dir = val
else:
print "Error: unknown ID in %s -> \"%s\"" % (worldname, val)
sys.exit(1)
# Now now load map files. These really should be named i_j.map but are
# actually j_i.map, presumably because of x_y.
for i in range(self.rows):
for j in range(self.cols):
key = "%s_%s.map" % (i, j)
key = os.path.join(self.dir, key)
# key should now be "dir/i_j.map"
# Load map into world dict
self.world[(i,j)] = key
self.terrain[key] = (i, j, None)
# now determine filenames for N/S/E/W maps. This replaces the
# old .world files
if j > 0:
self.north[key] = os.path.join(self.dir, "%d_%d.map" % (i, j-1))
else:
self.north[key] = None
if j < self.rows - 1:
self.south[key] = os.path.join(self.dir, "%d_%d.map" % (i, j+1))
else:
self.south[key] = None
if i > 0:
self.west[key] = os.path.join(self.dir, "%d_%d.map" % (i-1, j))
else:
self.west[key] = None
if i < self.cols - 1:
self.east[key] = os.path.join(self.dir, "%d_%d.map" % (i+1, j))
else:
self.east[key] = None
def __init__(self, worldname): f = util.loadMap(worldname) lines = f.readlines() for line in lines: line = line.rstrip() if line is not "": args = line.split() self.north[args[0]] = args[1] self.south[args[0]] = args[2] self.east[args[0]] = args[3] self.west[args[0]] = args[4]
def test(self,sess,X_test,X_test_str,output_path):
precision = []
recall = []
label2id,id2label = util.loadMap('data/label2id')
num_iterations = int(math.ceil(1.0 * len(X_test) / self.batch_size))
# print "number of iteration:" + str(num_iterations)
with open(output_path,'w') as outfiler:
for iteration in range(num_iterations):
# print 'iteration:' + str(iteration+1)
start_index = iteration*self.batch_size
end_index = min(start_index + self.batch_size,len(X_test))
X_test_batch = X_test[start_index:end_index]
X_test_batch_str = X_test_str[start_index:end_index]
if not self.is_crf:
logits_test = sess.run(
[
self.unary_scores
],
feed_dict =\
{
self.inputs:X_test_batch
}
)
y_pred = self.predict(logits_test[0],self.num_steps)
else:
unary_scores,transMatrix,test_seq_len = sess.run(
[
self.unary_scores,
self.transition_params,
self.length
],
feed_dict = \
{
self.inputs:X_test_batch
})
y_pred = self.viterbi_decode_batch(unary_scores,test_seq_len,transMatrix)
for batch_id in range(len(X_test_batch)):
for word_id in range(len(X_test_batch_str[batch_id])):
outfiler.write(X_test_batch_str[batch_id][word_id] + '\t' + id2label[y_pred[batch_id][word_id]]+'\n')
outfiler.write('\n')
def __init__(self, mapName):
for y in range(Map.NUM_ROWS):
self.atLayer.append(['.'] * Map.NUM_COLS)
self.belowLayer.append(['.'] * Map.NUM_COLS)
f = util.loadMap(mapName)
contents = f.readlines()
contents = self.clean(contents)
lineIndex = 0
self.mapName = contents[lineIndex]
lineIndex = lineIndex + 1 # the [RSC] tag
lineIndex = lineIndex + 1 # move to first resource
while contents[lineIndex] != "[AT]":
pair = contents[lineIndex].split(' ')
self.aliases[pair[0]]=pair[1]
lineIndex = lineIndex + 1 # move to next pair or [AT] tag
for y in range(24):
lineIndex = lineIndex + 1
for x in range(28):
self.atLayer[y][x] = contents[lineIndex][x]
lineIndex = lineIndex + 1 # the [BELOW] tag
for y in range(24):
lineIndex = lineIndex + 1
for x in range(28):
self.belowLayer[y][x] = contents[lineIndex][x]
lineIndex = lineIndex + 1 # the [ENTITY] tag
lineIndex = lineIndex + 1 # move to first entity
while lineIndex < len(contents):
self.entities.append(contents[lineIndex].split(' '))
lineIndex = lineIndex + 1
def train(self,sess,save_path,X_train,y_train,X_eval,y_val):
#X_train = [all_sentence_size,num_steps]
#y_train = [all_sentence_size,num_steps]
_,id2word = util.loadMap('data/word2id')
_,id2label = util.loadMap('data/label2id')
saver = tf.train.Saver()
num_iterations = int(math.ceil(1.0 * len(X_train) / self.batch_size))
# print y_train.shape
for epoch in range(self.num_epochs):
sh_index = np.arange(len(X_train))
np.random.shuffle(sh_index)
X_train = X_train[sh_index]
y_train = y_train[sh_index]
# print y_train.shape
print "current epoch: %d" % (epoch)
for iteration in range(num_iterations):
# X_train_batch = [batch_size,num_steps]
# y_train_batch = [batch_size,num_steps]
X_train_batch,y_train_batch = util.nextBatch(X_train,y_train,start_index = iteration*self.batch_size,batch_size = self.batch_size)
if not self.is_crf:
# y_train_batch = [batchsize,numsteps,numclasses]
y_train_batch = util.reshape_to_batchnumsteps_numclasses(y_train_batch, self.num_classess)
feed_dict = {self.inputs: X_train_batch,
self.targets: y_train_batch}
# \
if not self.is_crf:
_,unary_scores,lengths,loss_train\
= sess.run([
self.optimizer,
self.unary_scores,
self.length,
self.loss
],
feed_dict = feed_dict)
if iteration % 10 == 0:
y_batch_pred = self.predict(unary_scores, self.num_steps)
# y_batch_pred = self.nn_decode_batch(sess,unary_scores,lengths)
y_train_batch = util.reshape_to_batch_numsteps(y_train_batch, self.num_classess, self.num_steps)
precison, recall, f1 = self.evaluate(X_train_batch, y_train_batch, y_batch_pred, lengths,
id2word=id2word, id2label=id2label)
print 'evalution on train data is loss_train%f,precison%f,recall%f,f1%f' % (
loss_train, precison, recall, f1)
else:
_,unary_scores,lengths,loss_train,transMatrix\
= sess.run([
self.optimizer,
self.unary_scores,
self.length,
self.loss,
self.transition_params
],
feed_dict = feed_dict)
if iteration % 10 == 0:
y_batch_pred_crf = self.viterbi_decode_batch(unary_scores,lengths,transMatrix)
precison,recall,f1 = self.evaluate(X_train_batch,y_train_batch,y_batch_pred_crf,lengths,id2word=id2word,id2label=id2label)
print 'evalution on train data is loss_train%f,precison%f,recall%f,f1%f'%(loss_train,precison,recall,f1)
if iteration % 10 == 0:
# y_eval_pred = self.decode_all(sess,X_eval)
#y_val = [-1,num_steps]
#y_eval = [-1,num_steps,num_classes]
y_eval_pred,eval_loss,eval_seq_len = self.decode_all(sess,X_eval,y_val)
precison, recall, f1 = self.evaluate(X_eval, y_val, y_eval_pred, eval_seq_len, id2word=id2word,
id2label=id2label)
print 'evalution on eval data is eval_loss%f,precison%f,recall%f,f1%f' % (
eval_loss, precison, recall, f1)
if f1 > self.max_f1:
self.max_f1 = f1
saver.save(sess,save_path)
print 'the best model evaltion:f1:%f'%self.max_f1
num_steps = int(args.num_steps)
num_hiddens = int(args.num_hiddens)
num_layers = int(args.num_layers)
start_time = time.time()
X_eval = None
y_eval = None
print "prepare train and validation data"
#X_train,y_train = util.getTrainData(train_path,seq_max_len=num_steps)
X_train,y_train,X_eval,y_eval = util.getTrainEvalData(train_path = train_path, seq_max_len = num_steps)
print 'X_train.shape:'+str(X_train.shape)
print 'y_train.shape:'+str(y_train.shape)
if X_eval is not None:
print 'X_eval.shape:'+str(X_eval.shape)
word2id,_ = util.loadMap('data/word2id')
label2id,_ = util.loadMap('data/label2id')
num_words = len(word2id)
num_classes = len(label2id)
if not is_crf:
num_classes = num_classes-2
if emb_path != None:
embedding_matrix = util.getEmbedding(emb_path)
else:
embedding_matrix = None
# print y_eval
with tf.Session() as sess:
initializer = tf.random_uniform_initializer(-0.1,0.1)
with tf.variable_scope("model",reuse=None,initializer = initializer):
model = BILSTM(
num_words,
def __init__(self, mapName):
self.aliases = {}
self.atLayer = []
self.belowLayer = []
self.overLayer = []
self.entities = []
for y in range(Map.NUM_ROWS):
blanks = ['.'] * Map.NUM_COLS
self.atLayer.append(blanks[:])
self.belowLayer.append(blanks[:])
self.overLayer.append(blanks[:])
f = util.loadMap(mapName)
lines = self.clean(f.readlines())
f.close()
end = len(lines)
lineIndex = 0
self.mapName = lines[lineIndex]
lineIndex += 1
section = lines[lineIndex]
# Required RSC section
if section != "[RSC]":
raise Exception("Missing RSC Header in map file '%s'" % mapName)
lineIndex += 1
while lineIndex < end and lines[lineIndex] != "[AT]":
line = lines[lineIndex]
if line:
alias, image = line.split()
self.aliases[alias] = image
lineIndex += 1
# Required AT
if lineIndex >= end or lines[lineIndex] != "[AT]":
raise Exception("No [AT] group in map file '%s'" % mapName)
lineIndex += 1
for y in range(Map.NUM_ROWS):
for x in range(Map.NUM_COLS):
self.atLayer[y][x] = lines[lineIndex][x]
lineIndex += 1
# Required UNDER
while lineIndex < end and lines[lineIndex] != "[UNDER]":
lineIndex += 1
if lineIndex >= end or lines[lineIndex] != "[UNDER]":
raise Exception("No [UNDER] group in map file '%s'" % mapName)
lineIndex += 1
for y in range(Map.NUM_ROWS):
for x in range(Map.NUM_COLS):
self.belowLayer[y][x] = lines[lineIndex][x]
lineIndex += 1
while lineIndex < end and lines[lineIndex] not in ("[ENTITY]", "[OVER]"):
lineIndex += 1
# Optional OVER
if lineIndex < end and lines[lineIndex] == "[OVER]":
lineIndex += 1
for y in range(Map.NUM_ROWS):
for x in range(Map.NUM_COLS):
self.overLayer[y][x] = lines[lineIndex][x]
lineIndex += 1
# Optional ENTITY
while lineIndex < end and lines[lineIndex] not in "[ENTITY]":
lineIndex += 1
if lineIndex < end and lines[lineIndex] == "[ENTITY]":
lineIndex += 1
while lineIndex < end:
self.entities.append(lines[lineIndex].split(' '))
lineIndex += 1
