fw_cell = tf.nn.rnn_cell.DropoutWrapper(fw_cell, output_keep_prob=dropout)
lstm_fw_cell_m = tf.nn.rnn_cell.MultiRNNCell([fw_cell] * layers, state_is_tuple=True)
outputs,_=tf.nn.dynamic_rnn(lstm_fw_cell_m,x,dtype=tf.float32)
return outputs
x1=tf.placeholder(shape=[1,None,256],dtype=tf.float32)
x2=tf.placeholder(shape=[1,None,256],dtype=tf.float32)
isTraining=tf.placeholder(dtype=tf.bool)
char_embedding=ld.load_char_embedding()
word_embedding=ld.load_word_embedding()
topic_des=ld.load_topic_des().fillna("</s>")
question_topic=ld.load_question_topic()
with open("/media/ada/软件/BaiduNetdiskDownload/ieee_zhihu_cup/ieee_zhihu_cup/question_train_set.txt") as f:
question_des=f.readline().strip("\n").split("\t")
question_id=question_des[0]
question_ct=question_des[1].split(",")
question_wt=question_des[2].split(",")
question_cd=question_des[3].split(",")
question_wd=question_des[4].split(",")
topic_id=(question_topic.loc[question_topic['question_id']==int(question_id)][['topic_id']]).values
topic_id=topic_id.reshape([1])
def train(args):
file=open("/media/ada/软件/BaiduNetdiskDownload/ieee_zhihu_cup/ieee_zhihu_cup/question_eval.csv","a")
topic_info = pd.read_csv("/media/ada/软件/BaiduNetdiskDownload/ieee_zhihu_cup/ieee_zhihu_cup/topic_num.txt",
header=None, sep="\t")
char_embedding = ld.load_char_embedding()
args.vocab_size = char_embedding.shape[0]
global ckpt
if args.init_from is not None:
ckpt = tf.train.get_checkpoint_state(args.init_from)
model = Model(args,True)
#word_embedding = ld.load_word_embedding()
with tf.Session() as sess:
init = tf.global_variables_initializer()
sess.run(init)
saver = tf.train.Saver()
if args.init_from is not None:
saver.restore(sess, ckpt.model_checkpoint_path)
num = 0
header_list = []
for i in range(1, 257, 1):
header_list.append(str(i))
x_ = []
temp = []
for i in range(256):
temp.append(0)
with open("/media/ada/软件/BaiduNetdiskDownload/ieee_zhihu_cup/ieee_zhihu_cup/question_eval_set.txt","r") as f:
question_des = True
# for i in range(200): #已经训练了200代
# question_topic=f.readline().strip("\n").split("\t")
while (True):
question_des = f.readline().strip("\n").split("\t")
num += 1
if (num == 216467):
break
while (question_des):
question_des = f.readline().strip("\n").split("\t")
num+=1
question_id = question_des[0]
question_ct = question_des[1].split(",")
'''
question_wt = \
question_des.loc[question_des['question_id'] == int(question_id)][['question_name_word']].values[0][
0].split(",")
question_cd = \
question_des.loc[question_des['question_id'] == int(question_id)][['question_des_char']].values[0][
0].split(",")
question_wd = \
question_des.loc[question_des['question_id'] == int(question_id)][['question_des_word']].values[0][
0].split(",")
'''
question_ct_embedding = []
if(len(question_ct)>args.seq_length):
question_ct=question_ct[:args.seq_length]
for question_ct_ in question_ct:
if (len(char_embedding.loc[char_embedding['char'] == question_ct_]) == 0):
temp = np.random.rand(256)
question_ct_embedding.append(temp)
else:
question_ct_embedding.append(
char_embedding.loc[char_embedding['char'] == question_ct_][header_list].values[0])
while(len(question_ct_embedding)<args.seq_length):
question_ct_embedding.append(temp)
question_ct_embedding = np.mat(question_ct_embedding, dtype=np.float32)
x_.append(question_ct_embedding)
index=[-1 for i in range(5)]
index=np.array(index)
probablity=model.predict_class(sess, x_)[0]
priority=[-1 for j in range(5)]
for m in range(2000):
for n in range(5):
if(probablity[m]>priority[n]):
for k in range(3,n-1,-1):
priority[k+1]=priority[k]
index[k+1]=index[k]
index[n] = m
priority[n] = probablity[m]
break
print(num,question_id)
buffer=str(question_id)
#file.write(str(question_id))
for k__ in range(5):
#file.write(","+str(topic_info[1][index[k__]]))
buffer=buffer+(","+str(topic_info[1][index[k__]]))
#file.write("\n")
buffer+=("\n")
file.write(buffer)
x_=[]
if(num%10==0):
file.flush()
file.close()
def train(args):
char_embedding = ld.load_char_embedding()
args.vocab_size = char_embedding.shape[0]
global ckpt
if args.init_from is not None:
ckpt = tf.train.get_checkpoint_state(args.init_from)
model = Model(args)
#word_embedding = ld.load_word_embedding()
question_des = ld.load_question_des()
topic_des = ld.load_topic_des()
num = 0
with tf.Session() as sess:
init = tf.global_variables_initializer()
sess.run(init)
saver = tf.train.Saver()
if args.init_from is not None:
saver.restore(sess, ckpt.model_checkpoint_path)
x_ = []
y_ = []
temp = []
for i in range(256):
temp.append(0)
with open(
"/media/ada/软件/BaiduNetdiskDownload/ieee_zhihu_cup/ieee_zhihu_cup/question_topic_train_set.txt"
) as f:
while (True):
question_topic = f.readline().strip("\n").split("\t")
num += 1
if (num == 1):
break
# for i in range(200): #已经训练了200代
# question_topic=f.readline().strip("\n").split("\t")
while (question_topic):
question_topic = f.readline().strip("\n").split("\t")
num += 1
if (num == 500000):
break
question_id = question_topic[0]
topic_id = question_topic[1].split(",")
question_ct = \
question_des.loc[question_des['question_id'] == int(question_id)][['question_name_char']].values[0][
0].split(",")
'''
question_wt = \
question_des.loc[question_des['question_id'] == int(question_id)][['question_name_word']].values[0][
0].split(",")
question_cd = \
question_des.loc[question_des['question_id'] == int(question_id)][['question_des_char']].values[0][
0].split(",")
question_wd = \
question_des.loc[question_des['question_id'] == int(question_id)][['question_des_word']].values[0][
0].split(",")
'''
question_ct_embedding = []
if (len(question_ct) <= 10):
continue
if (len(question_ct) > args.seq_length):
question_ct = question_ct[:args.seq_length]
for question_ct_ in question_ct:
header_list = []
for i in range(1, 257, 1):
header_list.append(str(i))
if (len(char_embedding.loc[char_embedding['char'] ==
question_ct_]) == 0):
temp = np.random.rand(256)
question_ct_embedding.append(temp)
else:
question_ct_embedding.append(char_embedding.loc[
char_embedding['char'] == question_ct_]
[header_list].values[0])
while (len(question_ct_embedding) < args.seq_length):
question_ct_embedding.append(temp)
question_ct_embedding = np.mat(question_ct_embedding,
dtype=np.float32)
for topic_ in topic_id:
index = 0
#y_temp=np.zeros([2000])
while (True):
if (topic_des['topic_id'][index] == int(topic_)):
#y_temp[index] = 1
break
index += 1
x_.append(question_ct_embedding)
y_.append(index) #0,1,2,3
break
if (len(x_) == args.batch_size):
print(num)
x, y = x_, y_
e = num / 32 + 100000
sess.run(
tf.assign(model.lr,
args.learning_rate * (args.decay_rate**e)))
feed = {model.input_data: x, model.targets: y}
for i in range(3):
#start = time.time()
train_loss, state, _, accuracy = sess.run(
[
model.cost, model.final_state, model.optimizer,
model.accuracy
],
feed_dict=feed)
print(train_loss, accuracy)
print("prediction:", model.predict_class(sess, x_))
x_ = []
y_ = []
saver.save(sess, "./data_1/model.ckpt_1")
print("model saved,path:./data_1.model.ckpt_1")