def train_model1(sess, train_op, n_epoch, opt, model1, X_dataset, Y_dataset, Z_dataset, qvars, pvars, dvars, qvars_y, pvars_y, dvars_y):
train_gen_op_x = train_op.get_train_gen_op_x()
train_disc_op_x = train_op.get_train_disc_op_x()
train_gen_op_y = train_op.get_train_gen_op_y()
train_disc_op_y = train_op.get_train_disc_op_y()
x = model1.get_x()
y = model1.get_y()
z = model1.get_z()
d1 = model1.get_d1()
d2 = model1.get_d2()
FG_x = []
FG_y = []
FD_x = []
FD_y = []
DG_xz = DG()
DG_xz.initial()
DG_yz = DG()
DG_yz.initial()
dmb1 = [[1]*2]*128
dmb2 = [[0]*2]*128
for epoch in tqdm( range(n_epoch), total=n_epoch):
X_dataset = shuffle(X_dataset)
Y_dataset = shuffle(Y_dataset)
Z_dataset = shuffle(Z_dataset)
i = 0
#print(dmb)
for xmb, ymb, zmb in iter_data(X_dataset, Y_dataset, Z_dataset, size=batch_size):
#print(xmb)
i = i + 1
for _ in range(1):
f_d_x, _ = sess.run([model1.get_disc_loss_x(), train_disc_op_x], feed_dict={x: xmb, y:ymb, z:zmb, d1:dmb1, d2:dmb2})
for _ in range(5):
f_g_x, _ = sess.run([[model1.get_gen_loss_x(), model1.get_gen_loss_xz(), model1.get_cost_x(), model1.get_cost_xz()], train_gen_op_x], feed_dict={x: xmb, y:ymb, z:zmb, d1:dmb1, d2:dmb2})
FG_x.append(f_g_x)
FD_x.append(f_d_x)
for _ in range(1):
f_d_y, _ = sess.run([model1.get_disc_loss_y(), train_disc_op_y], feed_dict={x: xmb, y:ymb, z:zmb, d1:dmb1, d2:dmb2})
for _ in range(5):
f_g_y, _ = sess.run([[model1.get_gen_loss_y(), model1.get_gen_loss_yz(), model1.get_cost_y(), model1.get_cost_yz()], train_gen_op_y], feed_dict={x: xmb, y:ymb, z:zmb, d1:dmb1, d2:dmb2})
FG_y.append(f_g_y)
FD_y.append(f_d_y)
print_xz(epoch, i, f_d_x, f_g_x[0], f_g_x[1], f_g_x[2], f_g_x[3])
print_yz(epoch, i, f_d_y, f_g_y[0], f_g_y[1], f_g_y[2], f_g_y[3])
DG_xz.set_FD(FD_x)
DG_xz.set_FG(FG_x)
DG_yz.set_FD(FD_y)
DG_yz.set_FG(FG_y)
return sess, DG_xz, DG_yz, model1
def load_data(file, data_size=None, dynamic=True):
path = os.path.join(os.path.dirname(__file__), file)
df = pd.read_csv(path)
p = df['sentence1'].values[0:data_size]
h = df['sentence2'].values[0:data_size]
label = df['label'].values[0:data_size]
p, h, label = shuffle(p, h, label)
p_index, h_index = create_data(p, h)
p_seg = map(
lambda x: list(
jieba.cut(re.sub("[！，。？、~@#￥%&*（）.,:：|/`()_;+；…\\\\\\-\\s]", "", x)
)), p)
h_seg = map(
lambda x: list(
jieba.cut(re.sub("[！，。？、~@#￥%&*（）.,:：|/`()_;+；…\\\\\\-\\s]", "", x)
)), h)
# p_seg = map(lambda x: list(jieba.cut(x)), p)
# h_seg = map(lambda x: list(jieba.cut(x)), h)
p_vec = list(map(lambda x: w2v(x, dynamic), p_seg))
h_vec = list(map(lambda x: w2v(x, dynamic), h_seg))
# if not dynamic:
# p_vec = np.array(list(map(lambda x: w2v_process(x), p_vec)))
# h_vec = np.array(list(map(lambda x: w2v_process(x), h_vec)))
return p_index, h_index, p_vec, h_vec, label
def load_all_data(path, data_size=None, maxlen=35):
df = pd.read_csv(path)
p = df['sentence1'].values[0:data_size]
h = df['sentence2'].values[0:data_size]
label = df['label'].values[0:data_size]
p, h, label = shuffle(p, h, label)
p_c_index, h_c_index = char_index(p, h, maxlen=maxlen)
p_seg = list(map(lambda x: list(jieba.cut(x)), p))
h_seg = list(map(lambda x: list(jieba.cut(x)), h))
p_w_index, h_w_index = word_index(p_seg, h_seg, maxlen=maxlen)
# 判断是否有相同的词
same_word = []
for p_i, h_i in zip(p_w_index, h_w_index):
dic = {}
for i in p_i:
if i == 0:
break
dic[i] = dic.get(i, 0) + 1
for index, i in enumerate(h_i):
if i == 0:
same_word.append(0)
break
dic[i] = dic.get(i, 0) - 1
if dic[i] == 0:
same_word.append(1)
break
if index == len(h_i) - 1:
same_word.append(0)
return p_c_index, h_c_index, p_w_index, h_w_index, same_word, label
def load_all_data(path, data_size=None):
# model = Word2Vec.load('../output/word2vec/word2vec.model')
model = gensim.models.KeyedVectors.load(
r'E:\\DATA\\tencent\\ChineseEmbedding.bin', mmap='r')
df = pd.read_csv(path)
p = df['sentence1'].values[0:data_size]
h = df['sentence2'].values[0:data_size]
label = df['label'].values[0:data_size]
p, h, label = shuffle(p, h, label)
p_c_index, h_c_index = char_index(p, h)
p_seg = list(
map(
lambda x: list(
jieba.cut(
re.sub("[！，。？、~@#￥%&*（）.,:：|/`()_;+；…\\\\\\-\\s]", "", x))
), p))
h_seg = list(
map(
lambda x: list(
jieba.cut(
re.sub("[！，。？、~@#￥%&*（）.,:：|/`()_;+；…\\\\\\-\\s]", "", x))
), h))
p_w_index, h_w_index = word_index(p_seg, h_seg)
p_seg = map(lambda x: list(jieba.cut(x)), p)
h_seg = map(lambda x: list(jieba.cut(x)), h)
p_w_vec = list(map(lambda x: w2v(x, model), p_seg))
h_w_vec = list(map(lambda x: w2v(x, model), h_seg))
p_w_vec = list(map(lambda x: w2v_process(x), p_w_vec))
h_w_vec = list(map(lambda x: w2v_process(x), h_w_vec))
# 判断是否有相同的词
same_word = []
for p_i, h_i in zip(p_w_index, h_w_index):
dic = {}
for i in p_i:
if i == 0:
break
dic[i] = dic.get(i, 0) + 1
for index, i in enumerate(h_i):
if i == 0:
same_word.append(0)
break
dic[i] = dic.get(i, 0) - 1
if dic[i] == 0:
same_word.append(1)
break
if index == len(h_i) - 1:
same_word.append(0)
return p_c_index, h_c_index, p_w_index, h_w_index, p_w_vec, h_w_vec, same_word, label
def load_char_data(file, data_size=None):
path = os.path.join(os.path.dirname(__file__), '../' + file)
df = pd.read_csv(path)
p = df['sentence1'].values[0:data_size]
h = df['sentence2'].values[0:data_size]
label = df['label'].values[0:data_size]
p, h, label = shuffle(p, h, label)
p_c_index, h_c_index = char_index(p, h)
return p_c_index, h_c_index, label
def load_char_data(file, data_size=None, maxlen=35):
path = os.path.join(os.path.dirname(__file__), '../' + file)
df = pd.read_csv(path)
p = df['sentence1'].values[0:data_size]
h = df['sentence2'].values[0:data_size]
label = df['label'].values[0:data_size]
p, h, label = shuffle(p, h, label)
# [1,2,3,4,5] [4,1,5,2,0]
p_c_index, h_c_index = char_index(p, h, maxlen=maxlen)
return p_c_index, h_c_index, label
def train_model2(train_op, n_epoch_2, opt, model1, sess, X_dataset, Y_dataset, W_dataset, Z_dataset):
train_gen_op_xyw = train_op.get_train_gen_op_xyw()
train_disc_op_xyw = train_op.get_train_disc_op_xyw()
FD_xyw = []
FG_xyw = []
DG_xyw = DG()
DG_xyw.initial()
dmb1 = [[0]*2]*128
dmb2 = [[1]*2]*128
x = model1.get_x()
y = model1.get_y()
w = model1.get_w()
z = model1.get_z()
d1 = model1.get_d1()
d2 = model1.get_d2()
for epoch in tqdm( range(n_epoch_2), total=n_epoch_2):
X_dataset = shuffle(X_dataset)
Y_dataset = shuffle(Y_dataset)
W_dataset = shuffle(W_dataset)
Z_dataset = shuffle(Z_dataset)
i = 0
for xmb, ymb, wmb, zmb in iter_data(X_dataset, Y_dataset, W_dataset, Z_dataset, size=batch_size):
i = i + 1
for _ in range(1):
f_d_xyw, _ = sess.run([model1.get_disc_loss_xyw(), train_disc_op_xyw], feed_dict={x: xmb, y:ymb, w:wmb, z:zmb, d1:dmb1, d2:dmb2})
for _ in range(5):
f_g_xyw, _ = sess.run([[model1.get_gen_loss_xyw(), model1.get_gen_loss_x(), model1.get_cost_x(), model1.get_cost_xz()], train_gen_op_xyw], feed_dict={x: xmb, y:ymb, w:wmb, z:zmb, d1:dmb1, d2:dmb2})
FG_xyw.append(f_g_xyw)
FD_xyw.append(f_d_xyw)
print_xy(epoch, i, f_d_xyw, f_g_xyw[0], f_g_xyw[1], f_g_xyw[2], f_g_xyw[3])
DG_xyw.set_FD(FD_xyw)
DG_xyw.set_FG(FG_xyw)
return sess, DG_xyw, model1
def load_char_word_dynamic_data(path, data_size=None):
df = pd.read_csv(path)
p = df['sentence1'].values[0:data_size]
h = df['sentence2'].values[0:data_size]
label = df['label'].values[0:data_size]
p, h, label = shuffle(p, h, label)
p_char_index, h_char_index = char_index(p, h)
p_seg = map(lambda x: list(jieba.cut(re.sub("[！，。？、~@#￥%&*（）.,:：|/`()_;+；…\\\\\\-\\s]", "", x))), p)
h_seg = map(lambda x: list(jieba.cut(re.sub("[！，。？、~@#￥%&*（）.,:：|/`()_;+；…\\\\\\-\\s]", "", x))), h)
p_word_index, h_word_index = word_index(p_seg, h_seg)
return p_char_index, h_char_index, p_word_index, h_word_index, label
def load_data(file, data_size=None):
path = os.path.join(os.path.dirname(__file__), file)
df = pd.read_csv(path)
p = df['sentence1'].values[0:data_size]
h = df['sentence2'].values[0:data_size]
label = df['label'].values[0:data_size]
p, h, label = shuffle(p, h, label)
p_index, h_index = create_data(p, h)
p_seg = map(lambda x: list(jieba.cut(x)), p)
h_seg = map(lambda x: list(jieba.cut(x)), h)
p_vec = map(lambda x: w2v(x), p_seg)
h_vec = map(lambda x: w2v(x), h_seg)
p_vec = np.array(list(map(lambda x: w2v_process(x), p_vec)))
h_vec = np.array(list(map(lambda x: w2v_process(x), h_vec)))
return p_index, h_index, p_vec, h_vec, label
def load_char_word_static_data(file, data_size=None):
model = Word2Vec.load('../output/word2vec/word2vec.model')
path = os.path.join(os.path.dirname(__file__), file)
df = pd.read_csv(path)
p = df['sentence1'].values[0:data_size]
h = df['sentence2'].values[0:data_size]
label = df['label'].values[0:data_size]
p, h, label = shuffle(p, h, label)
p_c_index, h_c_index = char_index(p, h)
p_seg = map(lambda x: list(jieba.cut(x)), p)
h_seg = map(lambda x: list(jieba.cut(x)), h)
p_w_vec = list(map(lambda x: w2v(x, model), p_seg))
h_w_vec = list(map(lambda x: w2v(x, model), h_seg))
p_w_vec = list(map(lambda x: w2v_process(x), p_w_vec))
h_w_vec = list(map(lambda x: w2v_process(x), h_w_vec))
return p_c_index, h_c_index, p_w_vec, h_w_vec, label
FG_x = []
FG_y = []
FD_x = []
FD_y = []
FG_xy = []
FD_xy = []
X_dataset = data_x.get_dataset()
Y_dataset = data_y.get_dataset()
Z_dataset = data_z.get_dataset()
n_epoch = option.n_epoch
batch_size = option.batch_size
n_epoch_2 = option.n_epoch_2
for epoch in tqdm(range(n_epoch), total=n_epoch):
X_dataset = shuffle(X_dataset)
Y_dataset = shuffle(Y_dataset)
Z_dataset = shuffle(Z_dataset)
i = 0
for xmb, ymb, zmb in iter_data(X_dataset,
Y_dataset,
Z_dataset,
size=batch_size):
i = i + 1
for _ in range(1):
f_d_x, _ = sess.run([disc_loss_x, train_disc_op_x],
feed_dict={
x: xmb,
y: ymb,
z: zmb
})
opt = tf.train.AdamOptimizer(1e-3, beta1=0.5)
train_gen_op = opt.minimize(gen_loss, var_list=qvars + pvars)
train_disc_op = opt.minimize(disc_loss, var_list=dvars)
""" training """
config = tf.ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = 0.1
sess = tf.Session(config=config)
sess.run(tf.global_variables_initializer())
FG = []
FD = []
for epoch in tqdm( range(n_epoch), total=n_epoch):
X_dataset= shuffle(X_dataset)
Z_dataset= shuffle(Z_dataset)
i = 0
for xmb, zmb in iter_data(X_dataset, Z_dataset, size=batch_size):
i = i + 1
for _ in range(1):
f_d, _ = sess.run([disc_loss, train_disc_op], feed_dict={x: xmb, z:zmb})
for _ in range(5):
f_g, _ = sess.run([[gen_loss, gen_loss_xz, cost_x, cost_z], train_gen_op], feed_dict={x: xmb, z:zmb})
FG.append(f_g)
FD.append(f_d)
print("epoch %d iter %d: discloss %f genloss %f adv_x %f recons_x %f recons_z %f" % (epoch, i, f_d, f_g[0], f_g[1], f_g[2], f_g[3]))
# tmpx, tmpz = sess.run([dvars_x, dvars_z])
opt = tf.train.AdamOptimizer(1e-3, beta1=0.5)
train_gen_op = opt.minimize(gen_loss, var_list=px_vars + py_vars + x2y_vars + y2x_vars)
train_disc_op = opt.minimize(disc_loss, var_list=dvars)
""" training """
config = tf.ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = 0.1
sess = tf.Session(config=config)
sess.run(tf.global_variables_initializer())
FG = []
FD = []
for epoch in tqdm(range(n_epoch), total=n_epoch):
X_dataset= shuffle(X_dataset)
Y_dataset = shuffle(Y_dataset)
e1_dataset= shuffle(e1_dataset)
e2_dataset = shuffle(e2_dataset)
i = 0
for xmb, ymb, e1mb, e2mb in iter_data(X_dataset, Y_dataset, e1_dataset, e2_dataset, size=batch_size):
i = i + 1
for _ in range(1):
f_d, _ = sess.run([disc_loss, train_disc_op], feed_dict={x: xmb, y:ymb, e1:e1mb, e2:e2mb})
for _ in range(1):
f_g, _ = sess.run([[gen_loss, cost_x, cost_y], train_gen_op], feed_dict={x: xmb, y:ymb, e1:e1mb, e2:e2mb})
FG.append(f_g)
FD.append(f_d)
print("epoch %d iter %d: discloss %f genloss %f recons_x %f recons_y %f"
%(epoch, i, f_d, f_g[0], f_g[1], f_g[2]))
