def build_train_step(self):
# 损失
if self.loss is None:
_loss = Loss(label_data=self.label_data,
pred=self.pred,
output_act_func=self.output_act_func)
self.loss = _loss.get_loss_func(
self.loss_func
) # + 0.5*tf.matrix_determinant(tf.matmul(self.out_W,tf.transpose(self.out_W)))
# 正确率
if self.accuracy is None:
_ac = Accuracy(label_data=self.label_data, pred=self.pred)
self.accuracy = _ac.accuracy()
# 构建训练步
if self.train_batch is None:
if self.bp_algorithm == 'adam' or self.bp_algorithm == 'rmsp':
self.global_step = None
self.r = self.lr
else:
self.global_step = tf.Variable(
0, trainable=False) # minimize 中会对 global_step 自加 1
self.r = tf.train.exponential_decay(
learning_rate=self.lr,
global_step=self.global_step,
decay_steps=100,
decay_rate=0.96,
staircase=True)
self._optimization = Optimization(r=self.r, momentum=self.momentum)
self.train_batch = self._optimization.trainer(
algorithm=self.bp_algorithm).minimize(
self.loss, global_step=self.global_step)
def build_model(self):
print(self.name + ':')
print(self.__dict__)
# feed 变量
self.input_data = tf.placeholder(
tf.float32, [None, self.n_x],
name='X') # N等于batch_size（训练）或_num_examples（测试）
self.label_data = tf.placeholder(tf.float32, [None, self.n_x],
name='recon_X')
# 权值 变量（初始化）
self.W = tf.Variable(tf.truncated_normal(
shape=[self.n_x, self.n_h],
stddev=np.sqrt(2 / (self.n_x + self.n_h))),
name='W')
self.bz = tf.Variable(tf.constant(0.0, shape=[self.n_x]), name='bz')
self.bh = tf.Variable(tf.constant(0.0, shape=[self.n_h]), name='bh')
self.var_list = [self.W, self.bh, self.bz]
# 建模
x = self.input_data
if self.ae_type == 'dae': # 去噪自编码器 [dae]
x_ = self.add_noise(x)
h = self.transform(x_)
self.logist, self.pred = self.reconstruction(h)
self.loss = self.denoising_loss(x, self.pred)
else:
h = self.transform(x) # 自编码器 [ae]
self.logist, self.pred = self.reconstruction(h)
_loss = Loss(label_data=self.input_data,
pred=self.pred,
logist=self.logist,
output_act_func=self.de_func)
self.loss = _loss.get_loss_func(self.loss_func)
if self.ae_type == 'sae': # 稀疏自编码器 [sae]
self.loss = self.alpha * self.loss + self.beta * self.sparse_loss(
h)
# 构建训练步
self.build_train_step()
#****************** Tensorboard ******************
if self.tbd:
Summaries.scalars_histogram('_W', self.W)
Summaries.scalars_histogram('_bz', self.bz)
Summaries.scalars_histogram('_bh', self.bh)
tf.summary.scalar('loss', self.loss)
self.merge = tf.summary.merge(
tf.get_collection(tf.GraphKeys.SUMMARIES, self.name))
def build_model(self):
print(self.name + ':')
print(self.__dict__)
# feed 变量
self.input_data = tf.placeholder(
tf.float32, [None, self.n_x],
name='X') # N等于batch_size（训练）或_num_examples（测试）
self.label_data = tf.placeholder(tf.float32, [None, self.n_x],
name='Y')
self.A = tf.placeholder(tf.float32, [None, self.n_x], name='A')
# 权值 变量（初始化）
self.W = tf.Variable(tf.truncated_normal(shape=[self.n_x, self.n_y],
stddev=0.1),
name='W')
self.bz = tf.Variable(tf.constant(0.1, shape=[self.n_x]), name='bz')
self.by = tf.Variable(tf.constant(0.1, shape=[self.n_y]), name='by')
self.p_mat = tf.Variable(tf.constant(self.p, shape=[1, self.n_y]),
name='p_mat')
self.var_list = [self.W, self.by, self.bz]
# 建模
x = self.input_data
y = self.transform(x)
self.pred = self.reconstruction(y)
if self.ae_type == 'dae': # 去噪自编码器 [dae]
self.loss = self.get_denoising_loss(x, self.pred)
else:
_loss = Loss(
label_data=self.input_data, # 自编码器 [ae]
pred=self.pred,
output_act_func=self.de_func)
self.loss = _loss.get_loss_func(self.loss_func)
if self.ae_type == 'sae': # 稀疏自编码器 [sae]
self.loss = (1 -
self.beta) * self.loss + self.beta * self.KL(y)
# 构建训练步
self.build_train_step()
#****************** Tensorboard ******************
Summaries.scalars_histogram('_W', self.W)
Summaries.scalars_histogram('_bz', self.bz)
Summaries.scalars_histogram('_by', self.by)
tf.summary.scalar('loss', self.loss)
self.merge = tf.summary.merge(
tf.get_collection(tf.GraphKeys.SUMMARIES, self.name))
def build_model(self):
# feed 变量
self.input_data = tf.placeholder(
tf.float32, [None, self.n_x],
name='X') # N等于batch_size（训练）或_num_examples（测试）
self.A = tf.placeholder(tf.float32, [None, self.n_x], name='A')
# 权值 变量（初始化）
self.W = tf.Variable(tf.truncated_normal(shape=[self.n_x, self.n_y],
stddev=0.1),
name='W')
self.bz = tf.Variable(tf.constant(0.1, shape=[self.n_x]), name='bz')
self.by = tf.Variable(tf.constant(0.1, shape=[self.n_y]), name='by')
self.p_mat = tf.Variable(tf.constant(self.p, shape=[1, self.n_y]),
name='p_mat')
self.var_list = [self.W, self.by, self.bz]
# 建模
x = self.input_data
y = self.transform(x)
z = self.reconstruction(y)
if self.ae_type == 'dae': # 去噪自编码器 [dae]
self.loss = self.get_denoising_loss(x, z)
else:
_loss = Loss(
label_data=self.input_data, # 自编码器 [ae]
pred=z,
output_act_func=self.de_func)
self.loss = _loss.get_loss_func(self.loss_func)
if self.ae_type == 'sae': # 稀疏自编码器 [sae]
self.loss = (1 -
self.beta) * self.loss + self.beta * self.KL(y)
_optimization = Optimization(r=self.ae_lr,
momentum=self.momentum,
use_nesterov=True)
self.train_batch_bp = _optimization.trainer(algorithm='sgd').minimize(
self.loss, var_list=self.var_list)
#****************** Tensorboard ******************
Summaries.scalars_histogram('_W', self.W)
Summaries.scalars_histogram('_bz', self.bz)
Summaries.scalars_histogram('_by', self.by)
tf.summary.scalar('loss', self.loss)
self.merge = tf.summary.merge(
tf.get_collection(tf.GraphKeys.SUMMARIES,
tf.get_default_graph()._name_stack))
def build_model(self):
# feed 变量
self.input_data = tf.placeholder(
tf.float32,
[None, self.dbn_struct[0]]) # N等于batch_size（训练）或_num_examples（测试）
self.label_data = tf.placeholder(
tf.float32,
[None, self.dbn_struct[-1]]) # N等于batch_size（训练）或_num_examples（测试）
# 权值 变量（初始化）
self.out_W = tf.Variable(tf.truncated_normal(
shape=[self.dbn_struct[-2], self.dbn_struct[-1]], stddev=0.1),
name='W_out')
self.out_b = tf.Variable(tf.constant(0.1, shape=[self.dbn_struct[-1]]),
name='b_out')
# 构建rbms
self.rbms = RBMs(rbm_v_type=self.rbm_v_type,
rbms_struct=self.rbms_struct,
rbm_epochs=self.rbm_epochs,
batch_size=self.batch_size,
cd_k=self.cd_k,
rbm_lr=self.rbm_lr)
self.rbms.build_model()
# 构建dbn
# 构建权值列表（dbn结构）
self.parameter_list = list()
for rbm in self.rbms.rbm_list:
self.parameter_list.append(rbm.parameter)
self.parameter_list.append([self.out_W, self.out_b])
# 损失函数
self.pred = self.transform(self.input_data)
_loss = Loss(label_data=self.label_data,
pred=self.pred,
output_act_func=self.output_act_func)
self.loss = _loss.get_loss_func(self.loss_fuc)
self.train_batch_bp = tf.train.AdamOptimizer(
learning_rate=self.dbn_lr).minimize(self.loss,
var_list=self.parameter_list)
# 正确率
_ac = Accuracy(label_data=self.label_data, pred=self.pred)
self.accuracy = _ac.accuracy()
self.saver = tf.train.Saver()
def build_model(self):
"""
Pre-training
"""
# 构建dbm
self.dbm = DBM(rbm_v_type=self.rbm_v_type,
dbm_struct=self.dbm_struct,
rbm_epochs=self.rbm_epochs,
batch_size=self.batch_size,
cd_k=self.cd_k,
rbm_lr=self.rbm_lr)
"""
Fine-tuning
"""
with tf.name_scope('DBN'):
# feed 变量
self.input_data = tf.placeholder(
tf.float32, [None, self.dbn_struct[0]
]) # N等于batch_size（训练）或_num_examples（测试）
self.label_data = tf.placeholder(
tf.float32, [None, self.dbn_struct[-1]
]) # N等于batch_size（训练）或_num_examples（测试）
# 权值 变量（初始化）
self.out_W = tf.Variable(tf.truncated_normal(
shape=[self.dbn_struct[-2], self.dbn_struct[-1]], stddev=0.1),
name='W_out')
self.out_b = tf.Variable(tf.constant(0.0,
shape=[self.dbn_struct[-1]]),
name='b_out')
# 构建dbn
# 构建权值列表（dbn结构）
self.parameter_list = list()
for rbm in self.dbm.rbm_list:
self.parameter_list.append(rbm.W)
self.parameter_list.append(rbm.bh)
self.parameter_list.append(self.out_W)
self.parameter_list.append(self.out_b)
# 损失函数
self.pred = self.transform(self.input_data)
_loss = Loss(label_data=self.label_data,
pred=self.pred,
output_act_func=self.output_act_func)
self.loss = _loss.get_loss_func(self.loss_func)
_optimization = Optimization(r=self.dbn_lr, momentum=self.momentum)
self.train_batch_bp = _optimization.trainer(
algorithm=self.bp_algorithm).minimize(
self.loss, var_list=self.parameter_list)
# 正确率
_ac = Accuracy(label_data=self.label_data, pred=self.pred)
self.accuracy = _ac.accuracy()
#****************** 记录 ******************
for i in range(len(self.parameter_list)):
if i % 2 == 1: continue
k = int(i / 2 + 1)
W = self.parameter_list[i]
b = self.parameter_list[i + 1]
Summaries.scalars_histogram('_W' + str(k), W)
Summaries.scalars_histogram('_b' + str(k), b)
tf.summary.scalar('loss', self.loss)
tf.summary.scalar('accuracy', self.accuracy)
self.merge = tf.summary.merge(
tf.get_collection(tf.GraphKeys.SUMMARIES,
tf.get_default_graph()._name_stack))
def build_model(self):
# feed
self.input_data = tf.placeholder(tf.float32, [None, self.img_shape[0]*self.img_shape[1]]) # N等于batch_size（训练）或_num_examples（测试）
self.label_data = tf.placeholder(tf.float32, [None, self.channels[-1]]) # N等于batch_size（训练）或_num_examples（测试）
# reshape X
X = tf.reshape(self.input_data, shape=[-1,self.img_shape[0] , self.img_shape[1], self.channels[0]])
# conv,max_pool
self.W=list()
self.b=list()
c=0
p=0
for layer in self.layer_tp:
if layer=='C':
# W,b:conv
name_W='W_conv'+str(c+1)
name_b='b_conv'+str(c+1)
W_shape=[self.fsize[c][0],self.fsize[c][1],self.channels[c],self.channels[c+1]]
self.W.append(tf.Variable(tf.random_normal(shape=W_shape, stddev=0.1), name=name_W))
self.b.append(tf.Variable(tf.constant(0.1, shape=[self.channels[c+1]]),name=name_b))
# Tensorboard
Summaries.scalars_histogram('_'+name_W,self.W[-1])
Summaries.scalars_histogram('_'+name_b,self.b[-1])
# 卷积操作
"""transform < conv >"""
X = self.conv2d(X, self.W[-1], self.b[-1])
c=c+1
else:
k_shape=[1,self.ksize[p][0],self.ksize[p][1],1]
# 池化操作
"""transform < max_pool >"""
X = self.max_pool(X, ksize=k_shape)
p=p+1
# full_connect 全连接层
shape=X.get_shape().as_list()
full_size=shape[1]* shape[2]*shape[3]
X = tf.reshape(X, shape=[-1,full_size])
for i in range(c,len(self.channels)-1):
if i==c: n1=full_size
else: n1=self.channels[i]
n2=self.channels[i+1]
# W,b:full_connect
name_W='W_full'+str(i+1)
name_b='b_full'+str(i+1)
self.W.append(tf.Variable(tf.random_normal(shape=[n1,n2], stddev=0.1), name=name_W))
self.b.append(tf.Variable(tf.constant(0.1, shape=[n2]),name=name_b))
# Tensorboard
Summaries.scalars_histogram('_'+name_W,self.W[-1])
Summaries.scalars_histogram('_'+name_b,self.b[-1])
# 全连接
"""transform < full_connect >"""
z=tf.add(tf.matmul(X,self.W[-1]), self.b[-1])
if i==len(self.channels)-2:
self.pred = act_func(self.output_act_func)(z) # Relu activation
else:
X = act_func(self.hidden_act_func)(z) # Relu activation
if self.dropout<1:
X = tf.nn.dropout(X, self.dropout) # Apply Dropout
# loss,trainer
self.parameter_list=list()
for i in range(len(self.W)):
self.parameter_list.append(self.W[i])
self.parameter_list.append(self.b[i])
_loss=Loss(label_data=self.label_data,
pred=self.pred,
output_act_func=self.output_act_func)
self.loss=_loss.get_loss_func(self.loss_func)
self.train_batch_bp=tf.train.AdamOptimizer(learning_rate=self.cnn_lr).minimize(self.loss, var_list=self.parameter_list)
# accuracy
_ac=Accuracy(label_data=self.label_data,
pred=self.pred)
self.accuracy=_ac.accuracy()
# Tensorboard
tf.summary.scalar('loss',self.loss)
tf.summary.scalar('accuracy',self.accuracy)
self.merge = tf.summary.merge(tf.get_collection(tf.GraphKeys.SUMMARIES,tf.get_default_graph()._name_stack))
def build_model(self):
"""
Pre-training
"""
# 构建un_sae
self.un_sae = unsupervised_sAE(
en_func=self.en_func,
loss_func=self.loss_func, # encoder：[sigmoid] || decoder：[sigmoid] with ‘cross_entropy’ | [relu] with ‘mse’
ae_type=self.ae_type, # ae | dae | sae
noise_type=self.noise_type, # Gaussian noise (gs) | Masking noise (mn)
beta=self.beta, # 惩罚因子权重（第二项损失的系数）
p=self.p, # DAE：样本该维作为噪声的概率 / SAE稀疏性参数：期望的隐层平均活跃度（在训练批次上取平均）
un_ae_struct=self.un_ae_struct,
ae_epochs=self.ae_epochs,
batch_size=self.batch_size,
ae_lr=self.ae_lr)
"""
Fine-tuning
"""
with tf.name_scope('sup_sAE'):
# feed 变量
self.input_data = tf.placeholder(tf.float32, [None, self.sup_ae_struct[0]],name='X') # N等于batch_size（训练）或_num_examples（测试）
self.label_data = tf.placeholder(tf.float32, [None, self.sup_ae_struct[-1]],name='Y') # N等于batch_size（训练）或_num_examples（测试）
# 权值 变量（初始化）
self.out_W = tf.Variable(tf.truncated_normal(shape=[self.sup_ae_struct[-2], self.sup_ae_struct[-1]], stddev=0.1), name='W-out')
self.out_b = tf.Variable(tf.constant(0.1, shape=[self.sup_ae_struct[-1]]),name='b-out')
# 构建sup_sae
# 构建权值列表（sup_sae结构）
self.parameter_list = list()
for ae in self.un_sae.ae_list:
self.parameter_list.append(ae.W)
self.parameter_list.append(ae.by)
self.parameter_list.append(self.out_W)
self.parameter_list.append(self.out_b)
# 损失函数
self.pred=self.transform(self.input_data)
_loss=Loss(label_data=self.label_data,
pred=self.pred,
output_act_func=self.out_func)
self.loss=_loss.get_loss_func(self.loss_func)
_optimization=Optimization(r=self.ae_lr,
momentum=0.5)
self.train_batch_bp=_optimization.trainer(algorithm='sgd').minimize(self.loss, var_list=self.parameter_list)
# 正确率
_ac=Accuracy(label_data=self.label_data,
pred=self.pred)
self.accuracy=_ac.accuracy()
#****************** 记录 ******************
for i in range(len(self.parameter_list)):
if i%2==1:continue
k=int(i/2+1)
W=self.parameter_list[i]
b=self.parameter_list[i+1]
Summaries.scalars_histogram('_W'+str(k),W)
Summaries.scalars_histogram('_b'+str(k),b)
tf.summary.scalar('loss',self.loss)
tf.summary.scalar('accuracy',self.accuracy)
self.merge = tf.summary.merge(tf.get_collection(tf.GraphKeys.SUMMARIES,tf.get_default_graph()._name_stack))
def build_model(self):
# feed
self.input_data = tf.placeholder(
tf.float32, [None, self.img_shape[0] * self.img_shape[1]
]) # N等于batch_size（训练）或_num_examples（测试）
self.label_data = tf.placeholder(
tf.float32,
[None, self.channels[-1]]) # N等于batch_size（训练）或_num_examples（测试）
# reshape X
X = tf.reshape(
self.input_data,
shape=[-1, self.img_shape[0], self.img_shape[1], self.channels[0]])
# conv,max_pool
self.W = list()
self.b = list()
for i in range(len(self.channels) - 3):
str_w = 'W' + str(i)
str_b = 'b' + str(i)
w_shape = [
self.fsize[i][0], self.fsize[i][1], self.channels[i],
self.channels[i + 1]
]
k_shape = [1, self.ksize[i][0], self.ksize[i][1], 1]
# W,b:conv
self.W.append(
tf.Variable(tf.random_normal(shape=w_shape, stddev=0.1),
name=str_w))
self.b.append(
tf.Variable(tf.constant(0.1, shape=[self.channels[i + 1]]),
name=str_b))
"""transform <- conv,max_pool"""
conv = self.conv2d(X, self.W[i], self.b[i])
pool = self.max_pool(conv, ksize=k_shape)
X = tf.nn.dropout(pool, self.dropout)
# full_connect
shape = X.get_shape().as_list()
full_size = shape[1] * shape[2] * shape[3]
X = tf.reshape(X, shape=[-1, full_size])
# W,b:full_connect
self.W.append(
tf.Variable(tf.random_normal(shape=[full_size, self.channels[-2]],
stddev=0.1),
name='W_full'))
self.b.append(
tf.Variable(tf.constant(0.1, shape=[self.channels[-2]]),
name='b_full'))
"""transform <- full_connect"""
z = tf.add(tf.matmul(X, self.W[-1]), self.b[-1])
dense = self.func_h(z) # Relu activation
X = tf.nn.dropout(dense, self.dropout) # Apply Dropout
# classification
# W,b:classification
self.W.append(
tf.Variable(tf.random_normal(
shape=[self.channels[-2], self.channels[-1]], stddev=0.1),
name='W_out'))
self.b.append(
tf.Variable(tf.constant(0.1, shape=[self.channels[-1]]),
name='b_out'))
"""transform <- classification"""
z = tf.add(tf.matmul(X, self.W[-1]), self.b[-1])
self.pred = self.func_o(z)
# loss,trainer
self.parameter_list = [self.W, self.b]
_loss = Loss(label_data=self.label_data,
pred=self.pred,
output_act_func=self.output_act_func)
self.loss = _loss.get_loss_func(self.loss_fuc)
self.train_batch_bp = tf.train.AdamOptimizer(
learning_rate=self.cnn_lr).minimize(self.loss,
var_list=self.parameter_list)
# accuracy
_ac = Accuracy(label_data=self.label_data, pred=self.pred)
self.accuracy = _ac.accuracy()