def train_model1(sess, train_op, n_epoch, opt, model1, X_dataset, Y_dataset, Z_dataset, qvars, pvars, dvars, qvars_y, pvars_y, dvars_y): train_gen_op_x = train_op.get_train_gen_op_x() train_disc_op_x = train_op.get_train_disc_op_x() train_gen_op_y = train_op.get_train_gen_op_y() train_disc_op_y = train_op.get_train_disc_op_y() x = model1.get_x() y = model1.get_y() z = model1.get_z() d1 = model1.get_d1() d2 = model1.get_d2() FG_x = [] FG_y = [] FD_x = [] FD_y = [] DG_xz = DG() DG_xz.initial() DG_yz = DG() DG_yz.initial() dmb1 = [[1]*2]*128 dmb2 = [[0]*2]*128 for epoch in tqdm( range(n_epoch), total=n_epoch): X_dataset = shuffle(X_dataset) Y_dataset = shuffle(Y_dataset) Z_dataset = shuffle(Z_dataset) i = 0 #print(dmb) for xmb, ymb, zmb in iter_data(X_dataset, Y_dataset, Z_dataset, size=batch_size): #print(xmb) i = i + 1 for _ in range(1): f_d_x, _ = sess.run([model1.get_disc_loss_x(), train_disc_op_x], feed_dict={x: xmb, y:ymb, z:zmb, d1:dmb1, d2:dmb2}) for _ in range(5): f_g_x, _ = sess.run([[model1.get_gen_loss_x(), model1.get_gen_loss_xz(), model1.get_cost_x(), model1.get_cost_xz()], train_gen_op_x], feed_dict={x: xmb, y:ymb, z:zmb, d1:dmb1, d2:dmb2}) FG_x.append(f_g_x) FD_x.append(f_d_x) for _ in range(1): f_d_y, _ = sess.run([model1.get_disc_loss_y(), train_disc_op_y], feed_dict={x: xmb, y:ymb, z:zmb, d1:dmb1, d2:dmb2}) for _ in range(5): f_g_y, _ = sess.run([[model1.get_gen_loss_y(), model1.get_gen_loss_yz(), model1.get_cost_y(), model1.get_cost_yz()], train_gen_op_y], feed_dict={x: xmb, y:ymb, z:zmb, d1:dmb1, d2:dmb2}) FG_y.append(f_g_y) FD_y.append(f_d_y) print_xz(epoch, i, f_d_x, f_g_x[0], f_g_x[1], f_g_x[2], f_g_x[3]) print_yz(epoch, i, f_d_y, f_g_y[0], f_g_y[1], f_g_y[2], f_g_y[3]) DG_xz.set_FD(FD_x) DG_xz.set_FG(FG_x) DG_yz.set_FD(FD_y) DG_yz.set_FG(FG_y) return sess, DG_xz, DG_yz, model1
def load_data(file, data_size=None, dynamic=True): path = os.path.join(os.path.dirname(__file__), file) df = pd.read_csv(path) p = df['sentence1'].values[0:data_size] h = df['sentence2'].values[0:data_size] label = df['label'].values[0:data_size] p, h, label = shuffle(p, h, label) p_index, h_index = create_data(p, h) p_seg = map( lambda x: list( jieba.cut(re.sub("[!,。?、~@#¥%&*().,::|/`()_;+;…\\\\\\-\\s]", "", x) )), p) h_seg = map( lambda x: list( jieba.cut(re.sub("[!,。?、~@#¥%&*().,::|/`()_;+;…\\\\\\-\\s]", "", x) )), h) # p_seg = map(lambda x: list(jieba.cut(x)), p) # h_seg = map(lambda x: list(jieba.cut(x)), h) p_vec = list(map(lambda x: w2v(x, dynamic), p_seg)) h_vec = list(map(lambda x: w2v(x, dynamic), h_seg)) # if not dynamic: # p_vec = np.array(list(map(lambda x: w2v_process(x), p_vec))) # h_vec = np.array(list(map(lambda x: w2v_process(x), h_vec))) return p_index, h_index, p_vec, h_vec, label
def load_all_data(path, data_size=None, maxlen=35): df = pd.read_csv(path) p = df['sentence1'].values[0:data_size] h = df['sentence2'].values[0:data_size] label = df['label'].values[0:data_size] p, h, label = shuffle(p, h, label) p_c_index, h_c_index = char_index(p, h, maxlen=maxlen) p_seg = list(map(lambda x: list(jieba.cut(x)), p)) h_seg = list(map(lambda x: list(jieba.cut(x)), h)) p_w_index, h_w_index = word_index(p_seg, h_seg, maxlen=maxlen) # 判断是否有相同的词 same_word = [] for p_i, h_i in zip(p_w_index, h_w_index): dic = {} for i in p_i: if i == 0: break dic[i] = dic.get(i, 0) + 1 for index, i in enumerate(h_i): if i == 0: same_word.append(0) break dic[i] = dic.get(i, 0) - 1 if dic[i] == 0: same_word.append(1) break if index == len(h_i) - 1: same_word.append(0) return p_c_index, h_c_index, p_w_index, h_w_index, same_word, label
def load_all_data(path, data_size=None): # model = Word2Vec.load('../output/word2vec/word2vec.model') model = gensim.models.KeyedVectors.load( r'E:\\DATA\\tencent\\ChineseEmbedding.bin', mmap='r') df = pd.read_csv(path) p = df['sentence1'].values[0:data_size] h = df['sentence2'].values[0:data_size] label = df['label'].values[0:data_size] p, h, label = shuffle(p, h, label) p_c_index, h_c_index = char_index(p, h) p_seg = list( map( lambda x: list( jieba.cut( re.sub("[!,。?、~@#¥%&*().,::|/`()_;+;…\\\\\\-\\s]", "", x)) ), p)) h_seg = list( map( lambda x: list( jieba.cut( re.sub("[!,。?、~@#¥%&*().,::|/`()_;+;…\\\\\\-\\s]", "", x)) ), h)) p_w_index, h_w_index = word_index(p_seg, h_seg) p_seg = map(lambda x: list(jieba.cut(x)), p) h_seg = map(lambda x: list(jieba.cut(x)), h) p_w_vec = list(map(lambda x: w2v(x, model), p_seg)) h_w_vec = list(map(lambda x: w2v(x, model), h_seg)) p_w_vec = list(map(lambda x: w2v_process(x), p_w_vec)) h_w_vec = list(map(lambda x: w2v_process(x), h_w_vec)) # 判断是否有相同的词 same_word = [] for p_i, h_i in zip(p_w_index, h_w_index): dic = {} for i in p_i: if i == 0: break dic[i] = dic.get(i, 0) + 1 for index, i in enumerate(h_i): if i == 0: same_word.append(0) break dic[i] = dic.get(i, 0) - 1 if dic[i] == 0: same_word.append(1) break if index == len(h_i) - 1: same_word.append(0) return p_c_index, h_c_index, p_w_index, h_w_index, p_w_vec, h_w_vec, same_word, label
def load_char_data(file, data_size=None): path = os.path.join(os.path.dirname(__file__), '../' + file) df = pd.read_csv(path) p = df['sentence1'].values[0:data_size] h = df['sentence2'].values[0:data_size] label = df['label'].values[0:data_size] p, h, label = shuffle(p, h, label) p_c_index, h_c_index = char_index(p, h) return p_c_index, h_c_index, label
def load_char_data(file, data_size=None, maxlen=35): path = os.path.join(os.path.dirname(__file__), '../' + file) df = pd.read_csv(path) p = df['sentence1'].values[0:data_size] h = df['sentence2'].values[0:data_size] label = df['label'].values[0:data_size] p, h, label = shuffle(p, h, label) # [1,2,3,4,5] [4,1,5,2,0] p_c_index, h_c_index = char_index(p, h, maxlen=maxlen) return p_c_index, h_c_index, label
def train_model2(train_op, n_epoch_2, opt, model1, sess, X_dataset, Y_dataset, W_dataset, Z_dataset): train_gen_op_xyw = train_op.get_train_gen_op_xyw() train_disc_op_xyw = train_op.get_train_disc_op_xyw() FD_xyw = [] FG_xyw = [] DG_xyw = DG() DG_xyw.initial() dmb1 = [[0]*2]*128 dmb2 = [[1]*2]*128 x = model1.get_x() y = model1.get_y() w = model1.get_w() z = model1.get_z() d1 = model1.get_d1() d2 = model1.get_d2() for epoch in tqdm( range(n_epoch_2), total=n_epoch_2): X_dataset = shuffle(X_dataset) Y_dataset = shuffle(Y_dataset) W_dataset = shuffle(W_dataset) Z_dataset = shuffle(Z_dataset) i = 0 for xmb, ymb, wmb, zmb in iter_data(X_dataset, Y_dataset, W_dataset, Z_dataset, size=batch_size): i = i + 1 for _ in range(1): f_d_xyw, _ = sess.run([model1.get_disc_loss_xyw(), train_disc_op_xyw], feed_dict={x: xmb, y:ymb, w:wmb, z:zmb, d1:dmb1, d2:dmb2}) for _ in range(5): f_g_xyw, _ = sess.run([[model1.get_gen_loss_xyw(), model1.get_gen_loss_x(), model1.get_cost_x(), model1.get_cost_xz()], train_gen_op_xyw], feed_dict={x: xmb, y:ymb, w:wmb, z:zmb, d1:dmb1, d2:dmb2}) FG_xyw.append(f_g_xyw) FD_xyw.append(f_d_xyw) print_xy(epoch, i, f_d_xyw, f_g_xyw[0], f_g_xyw[1], f_g_xyw[2], f_g_xyw[3]) DG_xyw.set_FD(FD_xyw) DG_xyw.set_FG(FG_xyw) return sess, DG_xyw, model1
def load_char_word_dynamic_data(path, data_size=None): df = pd.read_csv(path) p = df['sentence1'].values[0:data_size] h = df['sentence2'].values[0:data_size] label = df['label'].values[0:data_size] p, h, label = shuffle(p, h, label) p_char_index, h_char_index = char_index(p, h) p_seg = map(lambda x: list(jieba.cut(re.sub("[!,。?、~@#¥%&*().,::|/`()_;+;…\\\\\\-\\s]", "", x))), p) h_seg = map(lambda x: list(jieba.cut(re.sub("[!,。?、~@#¥%&*().,::|/`()_;+;…\\\\\\-\\s]", "", x))), h) p_word_index, h_word_index = word_index(p_seg, h_seg) return p_char_index, h_char_index, p_word_index, h_word_index, label
def load_data(file, data_size=None): path = os.path.join(os.path.dirname(__file__), file) df = pd.read_csv(path) p = df['sentence1'].values[0:data_size] h = df['sentence2'].values[0:data_size] label = df['label'].values[0:data_size] p, h, label = shuffle(p, h, label) p_index, h_index = create_data(p, h) p_seg = map(lambda x: list(jieba.cut(x)), p) h_seg = map(lambda x: list(jieba.cut(x)), h) p_vec = map(lambda x: w2v(x), p_seg) h_vec = map(lambda x: w2v(x), h_seg) p_vec = np.array(list(map(lambda x: w2v_process(x), p_vec))) h_vec = np.array(list(map(lambda x: w2v_process(x), h_vec))) return p_index, h_index, p_vec, h_vec, label
def load_char_word_static_data(file, data_size=None): model = Word2Vec.load('../output/word2vec/word2vec.model') path = os.path.join(os.path.dirname(__file__), file) df = pd.read_csv(path) p = df['sentence1'].values[0:data_size] h = df['sentence2'].values[0:data_size] label = df['label'].values[0:data_size] p, h, label = shuffle(p, h, label) p_c_index, h_c_index = char_index(p, h) p_seg = map(lambda x: list(jieba.cut(x)), p) h_seg = map(lambda x: list(jieba.cut(x)), h) p_w_vec = list(map(lambda x: w2v(x, model), p_seg)) h_w_vec = list(map(lambda x: w2v(x, model), h_seg)) p_w_vec = list(map(lambda x: w2v_process(x), p_w_vec)) h_w_vec = list(map(lambda x: w2v_process(x), h_w_vec)) return p_c_index, h_c_index, p_w_vec, h_w_vec, label
FG_x = [] FG_y = [] FD_x = [] FD_y = [] FG_xy = [] FD_xy = [] X_dataset = data_x.get_dataset() Y_dataset = data_y.get_dataset() Z_dataset = data_z.get_dataset() n_epoch = option.n_epoch batch_size = option.batch_size n_epoch_2 = option.n_epoch_2 for epoch in tqdm(range(n_epoch), total=n_epoch): X_dataset = shuffle(X_dataset) Y_dataset = shuffle(Y_dataset) Z_dataset = shuffle(Z_dataset) i = 0 for xmb, ymb, zmb in iter_data(X_dataset, Y_dataset, Z_dataset, size=batch_size): i = i + 1 for _ in range(1): f_d_x, _ = sess.run([disc_loss_x, train_disc_op_x], feed_dict={ x: xmb, y: ymb, z: zmb })
opt = tf.train.AdamOptimizer(1e-3, beta1=0.5) train_gen_op = opt.minimize(gen_loss, var_list=qvars + pvars) train_disc_op = opt.minimize(disc_loss, var_list=dvars) """ training """ config = tf.ConfigProto() config.gpu_options.per_process_gpu_memory_fraction = 0.1 sess = tf.Session(config=config) sess.run(tf.global_variables_initializer()) FG = [] FD = [] for epoch in tqdm( range(n_epoch), total=n_epoch): X_dataset= shuffle(X_dataset) Z_dataset= shuffle(Z_dataset) i = 0 for xmb, zmb in iter_data(X_dataset, Z_dataset, size=batch_size): i = i + 1 for _ in range(1): f_d, _ = sess.run([disc_loss, train_disc_op], feed_dict={x: xmb, z:zmb}) for _ in range(5): f_g, _ = sess.run([[gen_loss, gen_loss_xz, cost_x, cost_z], train_gen_op], feed_dict={x: xmb, z:zmb}) FG.append(f_g) FD.append(f_d) print("epoch %d iter %d: discloss %f genloss %f adv_x %f recons_x %f recons_z %f" % (epoch, i, f_d, f_g[0], f_g[1], f_g[2], f_g[3])) # tmpx, tmpz = sess.run([dvars_x, dvars_z])
opt = tf.train.AdamOptimizer(1e-3, beta1=0.5) train_gen_op = opt.minimize(gen_loss, var_list=px_vars + py_vars + x2y_vars + y2x_vars) train_disc_op = opt.minimize(disc_loss, var_list=dvars) """ training """ config = tf.ConfigProto() config.gpu_options.per_process_gpu_memory_fraction = 0.1 sess = tf.Session(config=config) sess.run(tf.global_variables_initializer()) FG = [] FD = [] for epoch in tqdm(range(n_epoch), total=n_epoch): X_dataset= shuffle(X_dataset) Y_dataset = shuffle(Y_dataset) e1_dataset= shuffle(e1_dataset) e2_dataset = shuffle(e2_dataset) i = 0 for xmb, ymb, e1mb, e2mb in iter_data(X_dataset, Y_dataset, e1_dataset, e2_dataset, size=batch_size): i = i + 1 for _ in range(1): f_d, _ = sess.run([disc_loss, train_disc_op], feed_dict={x: xmb, y:ymb, e1:e1mb, e2:e2mb}) for _ in range(1): f_g, _ = sess.run([[gen_loss, cost_x, cost_y], train_gen_op], feed_dict={x: xmb, y:ymb, e1:e1mb, e2:e2mb}) FG.append(f_g) FD.append(f_d) print("epoch %d iter %d: discloss %f genloss %f recons_x %f recons_y %f" %(epoch, i, f_d, f_g[0], f_g[1], f_g[2]))