def build_fcn_net(self, inp, use_dice=False):
with self.graph.as_default():
self.saver = tf.train.Saver(max_to_keep=1)
with tf.name_scope("Out"):
bn1 = tf.layers.batch_normalization(inputs=inp, name='bn1')
dnn1 = tf.layers.dense(bn1, 200, activation=None, name='f1')
if use_dice:
dnn1 = dice(dnn1, name='dice_1')
else:
dnn1 = prelu(dnn1, 'prelu1')
dnn2 = tf.layers.dense(dnn1, 80, activation=None, name='f2')
if use_dice:
dnn2 = dice(dnn2, name='dice_2')
else:
dnn2 = prelu(dnn2, 'prelu2')
dnn3 = tf.layers.dense(dnn2, 2, activation=None, name='f3')
self.y_hat = tf.nn.softmax(dnn3) + 0.00000001
with tf.name_scope('Metrics'):
# Cross-entropy loss and optimizer initialization
# 'core_type_ph': [1, 1, 0,..],
ctr_loss = - tf.reduce_mean(tf.log(self.y_hat) * self.target_ph)
self.loss = ctr_loss
# tf.summary.scalar('loss', self.loss)
self.optimizer = tf.train.AdamOptimizer(learning_rate=self.lr_ph).minimize(self.loss)
# self.optimizer = tf.train.GradientDescentOptimizer(learning_rate=self.lr_ph).minimize(self.loss)
# Accuracy metric
self.accuracy = tf.reduce_mean(tf.cast(tf.equal(tf.round(self.y_hat), self.target_ph), tf.float32))
# tf.summary.scalar('accuracy', self.accuracy)
self.merged = tf.summary.merge_all()
def get_history_din_embedded(self):
# TODO: add mask info for this operation
his_days = [
'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine',
'ten', 'eleven', 'twelve', 'thirteen', 'fourteen'
]
for fir in his_days:
key = "history_" + fir + "_chap_ph"
embed_key = "history_" + fir + "_chap_embedded"
setattr(
self, embed_key,
get_mask_zero_embedded(self.chapters_embeddings_var,
getattr(self, key)))
for fir in his_days:
key = "history_" + fir + "_sec_ph"
embed_key = "history_" + fir + "_sec_embedded"
setattr(
self, embed_key,
get_mask_zero_embedded(self.sections_embeddings_var,
getattr(self, key)))
history_cha_embedded = []
history_sec_embedded = []
for fir in his_days[::-1]:
key_c = "history_" + fir + "_chap_embedded"
key_s = "history_" + fir + "_sec_embedded"
history_cha_embedded.append(
tf.reduce_mean(getattr(self, key_c), axis=-2))
history_sec_embedded.append(
tf.reduce_mean(getattr(self, key_s), axis=-2))
#self.history_all_embedded = tf.reshape(, [None,len(history_all_embedded),EMBEDDING_DIM])
# T*B*N -。 B*T*N
history_cha_emb = tf.transpose(history_cha_embedded, [1, 0, 2])
attention_cha_output = din_attention(tf.reduce_mean(
self.today_chapters_embedded, axis=-2),
history_cha_emb,
ATTENTION_SIZE,
self.history_mask_cha_ph,
stag="cha")
history_sec_emb = tf.transpose(history_sec_embedded, [1, 0, 2])
attention_sec_output = din_attention(tf.reduce_mean(
self.today_sections_embedded, axis=-2),
history_sec_emb,
ATTENTION_SIZE,
self.history_mask_sec_ph,
stag="sec")
att_fea1 = tf.reduce_sum(attention_cha_output, -2)
att_fea2 = tf.reduce_sum(attention_sec_output, -2)
#atte_out = tf.concat([att_fea1,att_fea2],axis=-1)
return att_fea1, att_fea2
def get_rnn_sum(input_seq,name='cha',nameln = 'sec'):
#with tf.name_scope("GRU"):
num_layers=2
HIDDEN_DIM=128
KEEP_PROB =0.8
with tf.name_scope('cell'):
def build_cell(n,m):
cell = tf.nn.rnn_cell.GRUCell(n)
cell = tf.nn.rnn_cell.DropoutWrapper(cell, output_keep_prob=m)
return cell
num_units=[HIDDEN_DIM//2,HIDDEN_DIM//2]
cell_fw = [build_cell(n,KEEP_PROB) for n in num_units]
cell_bw = [build_cell(n,KEEP_PROB) for n in num_units]
with tf.name_scope('gru'),tf.variable_scope("gru", reuse=tf.AUTO_REUSE):
biout,output_fw,output_bw = tf.contrib.rnn.stack_bidirectional_dynamic_rnn(
cell_fw,cell_bw,inputs= input_seq,dtype=tf.float32,scope=name)
#这里的 name 和上面 一样吗，还是 每个不一样，再加个name2
# lnout = tf.contrib.layers.layer_norm(
# inputs=(biout + input_seq), begin_norm_axis=-1, begin_params_axis=-1, scope=nameln)
lnout = tf.contrib.layers.layer_norm(
inputs=(biout + input_seq), begin_norm_axis=-1, begin_params_axis=-1)
rnnoutput = tf.reduce_mean(lnout, axis=-2)
#attention_output = layer_norm(biout + input_seq)
return rnnoutput
def get_history_sum_embedded(self):
# TODO: add mask info for this operation
his_days =['one', 'two', 'three', 'four','five','six','seven','eight','nine',
'ten','eleven','twelve','thirteen','fourteen']
for fir in his_days:
key_c = "history_" + fir + "_chap_ph"
embed_key_c = "history_" + fir + "_chap_embedded"
setattr(self, embed_key_c,get_mask_zero_embedded(self.chapters_embeddings_var,
getattr(self, key_c)))
key_s = "history_" + fir + "_sec_ph"
embed_key_s = "history_" + fir + "_sec_embedded"
setattr(self, embed_key_s,get_mask_zero_embedded(self.sections_embeddings_var,
getattr(self, key_s)))
key_st = "style_" + fir + "_ph"
embed_key_st = "history_" + fir + "_sty_embedded"
setattr(self, embed_key_st,tf.nn.embedding_lookup(self.style_embeddings_var,
getattr(self, key_st)))
chap = get_rnn_sum(self.history_one_chap_embedded,"rnncha")
#b*x*128 b*128 b*(128*14)
sec = get_rnn_sum(self.history_one_sec_embedded, "rnnsec")
sty = self.history_one_sty_embedded
for fir in his_days[:0:-1]:
key_c = "history_" + fir + "_chap_embedded"
chap = tf.concat([chap, get_rnn_sum(getattr(self, key_c), "rnncha")], axis=-1)
key_s = "history_" + fir + "_sec_embedded"
sec = tf.concat([sec, get_rnn_sum(getattr(self, key_s), "rnnsec")], axis=-1)
key_st = "history_" + fir + "_sty_embedded"
sty = tf.concat([sty, getattr(self, key_st)], axis=-1)
history_chap_emb = tf.reshape(chap, [-1, HIS_DAYS, EMBEDDING_DIM])
history_sec_emb = tf.reshape(sec, [-1, HIS_DAYS, EMBEDDING_DIM])
history_sty_emb = tf.reshape(sty, [-1, HIS_DAYS, EMBEDDING_DIM])
chap_mean = tf.reduce_mean(history_chap_emb, axis=-2)
sec_mean = tf.reduce_mean(history_sec_emb, axis=-2)
sty_mean = tf.reduce_mean(history_sty_emb, axis=-2)
#return chap_mean, sec_mean
return tf.concat([chap_mean, sec_mean,sty_mean], axis=-1)
def get_history_sum_embedded(self):
# TODO: add mask info for this operation
his_days =['one', 'two', 'three', 'four','five','six','seven','eight','nine',
'ten','eleven','twelve','thirteen','fourteen']
for fir in his_days:
key = "history_" + fir + "_chap_ph"
embed_key = "history_" + fir + "_chap_embedded"
setattr(self, embed_key,get_mask_zero_embedded(self.chapters_embeddings_var,
getattr(self, key)))
for fir in his_days:
key = "history_" + fir + "_sec_ph"
embed_key = "history_" + fir + "_sec_embedded"
setattr(self, embed_key,get_mask_zero_embedded(self.sections_embeddings_var,
getattr(self, key)))
chap = tf.reduce_mean(self.history_one_chap_embedded, axis=-2)
#b*x*128 b*128 b*(128*14)
sec = tf.reduce_mean(self.history_one_sec_embedded, axis=-2)
for fir in his_days[:0:-1]:
key_c = "history_" + fir + "_chap_embedded"
chap = tf.concat([chap, tf.reduce_mean(getattr(self, key_c), axis=-2)], axis=-1)
key_s = "history_" + fir + "_sec_embedded"
sec = tf.concat([sec, tf.reduce_mean(getattr(self, key_s), axis=-2)], axis=-1)
history_chap_emb = tf.reshape(chap, [-1, HIS_DAYS, EMBEDDING_DIM])
history_sec_emb = tf.reshape(sec, [-1, HIS_DAYS, EMBEDDING_DIM])
chap_mean = tf.reduce_mean(history_chap_emb, axis=-2)
sec_mean = tf.reduce_mean(history_sec_emb, axis=-2)
return chap_mean, sec_mean
def __init__(self, *, use_dice=False):
super().__init__(use_dice=use_dice)
self.other_inputs()
teacher = [
# self.teacher_id_embedded,
self.province_id_embedded,
self.city_id_embedded,
self.core_type_embedded,
self.student_count_embedded,
]
# 0-4
clazz = [
# self.class_id_embedded,
self.edition_id_embedded,
self.grade_id_embedded,
self.class_student_embedded,
self.cap_avg_embedded,
self.cap_max_embedded,
self.cap_min_embedded,
]
# 5-11
study = [
self.study_vector_embedded,
self.gap_days_embedded,
]
# 12-13
submit = [
self.month_submit_rate_embedded,
]
# 14
capacity = [
self.region_capacity_embedded,
]
# 15
prefer = [
self.prefer_assign_time_avg_embedded,
self.prefer_assign_time_var_embedded,
self.prefer_assign_rank_avg_embedded,
self.prefer_assign_rank_var_embedded,
]
# 16-19
register = [
self.register_diff_embedded,
]
# 20
homeworkcount = [
self.homework_count_embedded,
]
# 21
weekcount = [
self.week_count_embedded,
]
# 22
lastday = [
self.lastday_count_embedded,
]
# 23
o = teacher + clazz + study + submit + \
capacity + prefer + register + homeworkcount + weekcount + lastday
# here, we do like this, not tf.concat(o,axis=-1),
# because, issue:https://github.com/tensorflow/tensorflow/issues/24816
# ps: style,homework,reflect don't need to do like this, proved.
others = o[0]
for i in o[1:]:
others = tf.concat([others, i], axis=-1)
others = [others]
style = []
for fir in ["1", "2", "3", "4"]:
for sec in ["100", "010", "001", "110", "101", "011", "111"]:
embed_key = "style_" + fir + "0" + sec + "_embedded"
style.append(getattr(self, embed_key))
homework = []
homework.append(self.today_style_embedded)
homework.append(tf.reduce_mean(self.today_chapters_embedded, -2))
homework.append(tf.reduce_mean(self.today_sections_embedded, -2))
homework.append(self.history_chap_embedded)
homework.append(self.history_sec_embedded)
homework.append(self.history_chap_embedded *
tf.reduce_mean(self.today_chapters_embedded, -2))
homework.append(self.history_sec_embedded *
tf.reduce_mean(self.today_sections_embedded, -2))
reflect = []
reflect.append(tf.reduce_mean(self.reflect_value_embedded, -2))
with self.graph.as_default():
with tf.name_scope("Concat"):
inps = tf.concat(others + style + homework + reflect, -1)
self.build_fcn_net(inps, self.use_dice)
def get_history_gru_embedded(self):
# TODO: add mask info for this operation
his_days = [
'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine',
'ten', 'eleven', 'twelve', 'thirteen', 'fourteen'
]
for fir in his_days:
key = "history_" + fir + "_chap_ph"
embed_key = "history_" + fir + "_chap_embedded"
setattr(
self, embed_key,
get_mask_zero_embedded(self.chapters_embeddings_var,
getattr(self, key)))
for fir in his_days:
key = "history_" + fir + "_sec_ph"
embed_key = "history_" + fir + "_sec_embedded"
setattr(
self, embed_key,
get_mask_zero_embedded(self.sections_embeddings_var,
getattr(self, key)))
# [1,2][3][4,3,5][0,0]... chap 14天
# [][][] sec
# cha_sec 拼接后rnn cha rnn sec rnn 拼劲 或者不拼接
#
history_cha_embedded = []
history_sec_embedded = []
for fir in his_days[::-1]:
key_c = "history_" + fir + "_chap_embedded"
key_s = "history_" + fir + "_sec_embedded"
# B 3 128 B 128 14 B 128 B 14 128
history_cha_embedded.append(
tf.reduce_mean(getattr(self, key_c), axis=-2))
history_sec_embedded.append(
tf.reduce_mean(getattr(self, key_s), axis=-2))
history_cha_emb = tf.transpose(history_cha_embedded, [1, 0, 2])
history_sec_emb = tf.transpose(history_sec_embedded, [1, 0, 2])
with tf.name_scope("GRU"):
num_layers = 2
HIDDEN_DIM = 128
KEEP_PROB = 0.8
with tf.name_scope('gru1'):
def get_cell():
cell2 = tf.nn.rnn_cell.GRUCell(HIDDEN_DIM)
cell2_ = tf.nn.rnn_cell.DropoutWrapper(
cell2, output_keep_prob=KEEP_PROB)
return cell2_
cells = [get_cell() for _ in range(num_layers)]
Cell = tf.nn.rnn_cell.MultiRNNCell(cells, state_is_tuple=True)
rnnoutput, _ = tf.nn.dynamic_rnn(cell=Cell,
inputs=history_cha_emb,
dtype=tf.float32,
scope='cha')
rnnoutput1 = tf.reduce_sum(rnnoutput, axis=-2)
with tf.name_scope("GRU2"):
num_layers = 2
HIDDEN_DIM = 128
KEEP_PROB = 0.8
with tf.name_scope('gru2'):
cells2 = []
for _ in range(num_layers):
cell = tf.nn.rnn_cell.GRUCell(HIDDEN_DIM)
cell_ = tf.nn.rnn_cell.DropoutWrapper(
cell, output_keep_prob=KEEP_PROB)
cells2.append(cell_)
#cells2 = [get_cell2() for _ in range(num_layers)]
Cell2 = tf.nn.rnn_cell.MultiRNNCell(cells2, state_is_tuple=True)
rnnoutput2, _ = tf.nn.dynamic_rnn(cell=Cell2,
inputs=history_sec_emb,
dtype=tf.float32,
scope='SEC')
rnnoutput3 = tf.reduce_sum(rnnoutput2, axis=-2)
#gru_out = tf.concat([rnnoutput1,rnnoutput3],axis=-1)
return rnnoutput1, rnnoutput3
def __init__(self, dat, dim_rec, dim_z, dim_gen, scope='vae'):
assert 2 == dat.ndim
assert isinstance(dim_rec, tuple)
assert isinstance(dim_z, int)
assert isinstance(dim_gen, tuple)
init_w = tf.variance_scaling_initializer(scale=2.0,
mode='fan_in',
distribution='uniform')
init_b = tf.constant_initializer(0.01)
init_z = tf.zeros_initializer()
with tf.variable_scope(scope):
dat = self.dat = tf.constant(name='dat', value=dat)
bs_ = self.bs_ = tf.placeholder(name='bs_',
dtype=tf.int32,
shape=())
bat = self.bat = tf.random_uniform(name='bat',
shape=(bs_, ),
minval=0,
maxval=dat.shape[0],
dtype=tf.int32)
h = x = self.x = tf.nn.embedding_lookup(name='x',
params=dat,
ids=bat)
for i, dim in enumerate(dim_rec, 1):
name = "hr{}".format(i)
h = tf.layers.dense(name=name,
inputs=h,
units=dim,
activation=tf.nn.relu,
kernel_initializer=init_w,
bias_initializer=init_b)
setattr(self, name, h)
mu = self.mu = tf.layers.dense(name='mu',
inputs=h,
units=dim_z,
kernel_initializer=init_w,
bias_initializer=init_z)
lv = self.lv = tf.layers.dense(name='lv',
inputs=h,
units=dim_z,
kernel_initializer=init_w,
bias_initializer=init_z)
with tf.name_scope('z'):
h = z = self.z = mu + tf.exp(
0.5 * lv) * tf.random_normal(shape=tf.shape(lv))
for i, dim in enumerate(dim_gen, 1):
name = "hg{}".format(i)
h = tf.layers.dense(name=name,
inputs=h,
units=dim,
activation=tf.nn.relu,
kernel_initializer=init_w,
bias_initializer=init_b)
setattr(self, name, h)
logits = tf.layers.dense(
name='logits',
inputs=h,
units=dat.shape[1]
# , activation= tf.nn.sigmoid
,
kernel_initializer=init_w,
bias_initializer=init_z)
g = self.g = tf.sigmoid(logits)
with tf.name_scope('loss_recons'):
# loss_recons = self.loss_recons = tf.reduce_mean(
# tf.reduce_sum(tf.nn.sigmoid_cross_entropy_with_logits(labels= x, logits= logits), axis= 1))
loss_recons = self.loss_recons = tf.reduce_mean(
tf.reduce_sum(tf.square(x - g), axis=1))
with tf.name_scope('loss_relent'):
# loss_relent = self.loss_relent = tf.reduce_mean(
# 0.5 * tf.reduce_sum((- 1.0 - lv + tf.exp(lv) + tf.square(mu)), axis= 1))
loss_relent = self.loss_relent = tf.reduce_mean(
tf.reduce_sum((-1.0 - lv + tf.exp(lv) + tf.square(mu)),
axis=1))
with tf.name_scope('loss'):
loss = self.loss = loss_relent + loss_recons
up = self.up = tf.train.AdamOptimizer().minimize(loss)
self.step = 0
def get_history_din_embedded(self):
# TODO: add mask info for this operation
his_days = [
'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine',
'ten', 'eleven', 'twelve', 'thirteen', 'fourteen'
]
for fir in his_days:
key = "history_" + fir + "_chap_ph"
embed_key = "history_" + fir + "_chap_embedded"
setattr(
self, embed_key,
get_mask_zero_embedded(self.chapters_embeddings_var,
getattr(self, key)))
for fir in his_days:
key = "history_" + fir + "_sec_ph"
embed_key = "history_" + fir + "_sec_embedded"
setattr(
self, embed_key,
get_mask_zero_embedded(self.sections_embeddings_var,
getattr(self, key)))
history_cha_embedded = []
history_sec_embedded = []
for fir in his_days[::-1]:
key_c = "history_" + fir + "_chap_embedded"
key_s = "history_" + fir + "_sec_embedded"
history_cha_embedded.append(
tf.reduce_mean(getattr(self, key_c), axis=-2))
history_sec_embedded.append(
tf.reduce_mean(getattr(self, key_s), axis=-2))
#self.history_all_embedded = tf.reshape(, [None,len(history_all_embedded),EMBEDDING_DIM])
# T*B*N -。 B*T*N
history_cha_emb = tf.transpose(history_cha_embedded, [1, 0, 2])
#dien
with tf.name_scope('rnn_1'):
rnn_outputs, _ = dynamic_rnn(GRUCell(HIDDEN_SIZE),
inputs=history_cha_emb,
sequence_length=self.history_len_ph,
dtype=tf.float32,
scope="gru1")
with tf.name_scope('Attention_layer_1'):
att_outputs, alphas = din_fcn_attention(tf.reduce_mean(
self.today_chapters_embedded, axis=-2),
rnn_outputs,
ATTENTION_SIZE,
self.history_mask_cha_ph,
scope="1_1",
softmax_stag=1,
stag='1_1',
mode='LIST',
return_alphas=True)
with tf.name_scope('rnn_2'):
rnn_outputs2, final_state2 = dynamic_rnn(
VecAttGRUCell(HIDDEN_SIZE),
inputs=rnn_outputs,
att_scores=tf.expand_dims(alphas, -1),
sequence_length=self.history_len_ph,
dtype=tf.float32,
scope="gru2")
history_sec_emb = tf.transpose(history_sec_embedded, [1, 0, 2])
with tf.name_scope('rnn_11'):
rnn_outputs3, _ = dynamic_rnn(GRUCell(HIDDEN_SIZE),
inputs=history_sec_emb,
sequence_length=self.history_len_ph,
dtype=tf.float32,
scope="gru11")
with tf.name_scope('Attention_layer_11'):
att_outputs3, alphas3 = din_fcn_attention(tf.reduce_mean(
self.today_sections_embedded, axis=-2),
rnn_outputs3,
ATTENTION_SIZE,
self.history_mask_sec_ph,
scope="2_2",
softmax_stag=1,
stag='2_2',
mode='LIST',
return_alphas=True)
with tf.name_scope('rnn_22'):
rnn_outputs4, final_state3 = dynamic_rnn(
VecAttGRUCell(HIDDEN_SIZE),
inputs=rnn_outputs3,
att_scores=tf.expand_dims(alphas3, -1),
sequence_length=self.history_len_ph,
dtype=tf.float32,
scope="gru22")
return final_state2, final_state3
def get_history_bgru_embedded(self):
# TODO: add mask info for this operation
his_days =['one', 'two', 'three', 'four','five','six','seven','eight','nine',
'ten','eleven','twelve','thirteen','fourteen']
for fir in his_days:
key = "history_" + fir + "_chap_ph"
embed_key = "history_" + fir + "_chap_embedded"
setattr(self, embed_key,get_mask_zero_embedded(self.chapters_embeddings_var,
getattr(self, key)))
for fir in his_days:
key = "history_" + fir + "_sec_ph"
embed_key = "history_" + fir + "_sec_embedded"
setattr(self, embed_key,get_mask_zero_embedded(self.sections_embeddings_var,
getattr(self, key)))
history_cha_embedded = []
history_sec_embedded = []
for fir in his_days[::-1]:
key_c = "history_" + fir + "_chap_embedded"
key_s = "history_" + fir + "_sec_embedded"
# B 3 128 B 128 14 B 128 B 14 128
history_cha_embedded.append(
tf.reduce_mean(getattr(self, key_c),axis=-2))
history_sec_embedded.append(
tf.reduce_mean(getattr(self, key_s),axis=-2 ))
history_cha_emb = tf.transpose(history_cha_embedded, [1, 0, 2])
history_sec_emb = tf.transpose(history_sec_embedded, [1, 0, 2])
with tf.name_scope("GRU"):
num_layers=2
HIDDEN_DIM=128
KEEP_PROB =0.8
with tf.name_scope('cell'):
def build_cell(n,m):
cell = tf.nn.rnn_cell.GRUCell(n)
cell = tf.nn.rnn_cell.DropoutWrapper(cell, output_keep_prob=m)
return cell
num_units=[HIDDEN_DIM//2,HIDDEN_DIM//2]
cell_fw = [build_cell(n,KEEP_PROB) for n in num_units]
cell_bw = [build_cell(n,KEEP_PROB) for n in num_units]
with tf.name_scope('gru'):
biout,output_fw,output_bw = tf.contrib.rnn.stack_bidirectional_dynamic_rnn(
cell_fw,cell_bw,inputs= history_cha_emb,dtype=tf.float32
,scope='cha')
rnnoutput = tf.reduce_sum(biout, axis=-2)
with tf.name_scope('cell2'):
def build_cell2(n,m):
cell2 = tf.nn.rnn_cell.GRUCell(n)
cell2 = tf.nn.rnn_cell.DropoutWrapper(cell2, output_keep_prob=m)
return cell2
num_units2=[HIDDEN_DIM//2,HIDDEN_DIM//2]
cell_fw2 = [build_cell2(n,KEEP_PROB) for n in num_units2]
cell_bw2 = [build_cell2(n,KEEP_PROB) for n in num_units2]
with tf.name_scope('gru2'):
biout2,output_fw2,output_bw2 = tf.contrib.rnn.stack_bidirectional_dynamic_rnn(
cell_fw2,cell_bw2,inputs= history_sec_emb,dtype=tf.float32
,scope='sec')
rnnoutput2 = tf.reduce_sum(biout2, axis=-2)
#rnn_output = tf.concat([rnnoutput,rnnoutput2],axis=-1)
return rnnoutput,rnnoutput2
