def main(_):
word_char = 'word' # 'word' or 'char'
print('use word or char:',word_char)
FLAGS.file_name = word_char+'_'+FLAGS.file_name
print('model_path:',FLAGS.file_name)
model_path = os.path.join('models', FLAGS.file_name)
if os.path.exists(model_path) is False:
os.makedirs(model_path)
if FLAGS.file_name[-1] == '2':
from model2 import Model
elif FLAGS.file_name[-1] == '3':
from model3 import Model
elif FLAGS.file_name[-1] == '4':
from model4 import Model
elif FLAGS.file_name[-1] == '5':
from model5 import Model
else:
from model1 import Model
data_path,save_path = 'data','process_data1'
converter = TextConverter(word_char, data_path, save_path, FLAGS.num_steps)
embeddings = converter.embeddings
if word_char == 'word':
train_pkl = 'train_word.pkl'
val_pkl = 'val_word.pkl'
if word_char == 'char':
train_pkl = 'train_char.pkl'
val_pkl = 'val_char.pkl'
train_samples = converter.load_obj(os.path.join(save_path, train_pkl))
train_g = batch_generator(train_samples, FLAGS.batch_size)
val_samples = converter.load_obj(os.path.join(save_path, val_pkl))
val_g = val_samples_generator(val_samples)
print('use embeding:',FLAGS.use_embedding)
print('vocab size:',converter.vocab_size)
model = Model(converter.vocab_size,FLAGS,test=False, embeddings=embeddings)
# 继续上一次模型训练
FLAGS.checkpoint_path = tf.train.latest_checkpoint(model_path)
if FLAGS.checkpoint_path:
model.load(FLAGS.checkpoint_path)
model.train(train_g,
FLAGS.max_steps,
model_path,
FLAGS.save_every_n,
FLAGS.log_every_n,
val_g
)
def main(_):
# # FLAGS.start_string = FLAGS.start_string#.decode('utf-8')
word_char = 'word' # 'word' or 'char'
print('use word or char:', word_char)
FLAGS.file_name = word_char + '_' + FLAGS.file_name
print('model_path:', FLAGS.file_name)
model_path = os.path.join('models', FLAGS.file_name)
if os.path.isdir(model_path):
FLAGS.checkpoint_path = tf.train.latest_checkpoint(model_path)
if FLAGS.file_name[-1] == '2':
from model2 import Model
elif FLAGS.file_name[-1] == '3':
from model3 import Model
elif FLAGS.file_name[-1] == '4':
from model4 import Model
elif FLAGS.file_name[-1] == '5':
from model5 import Model
else:
from model1 import Model
data_path, save_path = 'data', 'process_data1'
converter = TextConverter(word_char, data_path, save_path, FLAGS.num_steps)
embeddings = converter.embeddings
if word_char == 'word':
test_pkl = 'test_word.pkl'
if word_char == 'char':
test_pkl = 'test_char.pkl'
test_samples = converter.load_obj(os.path.join(save_path, test_pkl))
print('use embeding:', FLAGS.use_embedding)
print('vocab size:', converter.vocab_size)
with open(model_path + '/submission.csv', 'w') as file:
file.write(str('y_pre') + '\n')
for i in range(0, len(test_samples), 5000): # 内存不足 分批test
print('>>>>:', i, '/', len(test_samples))
test_g = test_samples_generator(test_samples[i:i + 5000])
model = Model(converter.vocab_size,
FLAGS,
test=False,
embeddings=embeddings)
model.load(FLAGS.checkpoint_path)
model.test(test_g, model_path)
print('finished!')