def __init__(self,
output_act_func='softmax',
hidden_act_func='relu',
loss_func='cross_entropy',
use_for='classification',
bp_algorithm='adam',
lr=1e-3,
momentum=0.9,
epochs=100,
width=5,
struct=[784, 100, 10],
batch_size=32,
dropout=0):
Model.__init__(self, 'RNN')
self.output_act_func = output_act_func
self.hidden_act_func = hidden_act_func
self.loss_func = loss_func
self.use_for = use_for
self.bp_algorithm = bp_algorithm
self.lr = lr
self.momentum = momentum
self.epochs = epochs
self.width = width
self.struct = struct
self.batch_size = batch_size
self.dropout = dropout
if output_act_func == 'gauss':
self.loss_func = 'mse'
self.hidden_act = act_func(self.hidden_act_func)
self.output_act = act_func(self.output_act_func)
self.build_model()
def __init__(self,
out_func='softmax',
en_func='sigmoid', # encoder：[sigmoid]
use_for='classification',
loss_func='mse', # decoder：[sigmoid] with ‘cross_entropy’ | [relu] with ‘mse’
ae_type='ae', # ae | dae | sae
noise_type='gs', # Gaussian noise (gs) | Masking noise (mn)
beta=0.5, # 惩罚因子权重（第二项损失的系数）
p=0.5, # DAE：样本该维作为噪声的概率 / SAE稀疏性参数：期望的隐层平均活跃度（在训练批次上取平均）
sup_ae_struct=[784, 100, 100, 10],
sup_ae_epochs=100,
ae_epochs=10,
batch_size=32,
ae_lr=1e-3,
dropout=1):
self.out_func=out_func
self.en_func=en_func
self.use_for=use_for
self.loss_func=loss_func
self.ae_type = ae_type
self.noise_type = noise_type
self.beta = beta
self.p = p
self.sup_ae_struct = sup_ae_struct
self.un_ae_struct = sup_ae_struct[:-1]
self.sup_ae_epochs=sup_ae_epochs
self.ae_epochs = ae_epochs
self.batch_size = batch_size
self.ae_lr = ae_lr
self.dropout = dropout
self.hidden_act=act_func(self.en_func)
self.output_act=act_func(self.out_func)
self.build_model()
def transform(self,X):
# conv,max_pool
self.W=list()
self.b=list()
c=0
p=0
for layer in self.layer_tp:
if layer=='C':
# W,b:conv
name_W='W_conv'+str(c+1)
name_b='b_conv'+str(c+1)
W_shape=[self.fsize[c][0],self.fsize[c][1],self.channels[c],self.channels[c+1]]
self.W.append(tf.Variable(tf.random_normal(shape=W_shape, stddev=0.1), name=name_W))
self.b.append(tf.Variable(tf.constant(0.1, shape=[self.channels[c+1]]),name=name_b))
# 卷积操作
"""transform < conv >"""
X = self.conv2d(X, self.W[-1], self.b[-1])
c=c+1
else:
k_shape=[1,self.ksize[p][0],self.ksize[p][1],1]
# 池化操作
"""transform < max_pool >"""
X = self.max_pool(X, ksize=k_shape)
p=p+1
# full_connect 全连接层
shape=X.get_shape().as_list()
full_size=shape[1]* shape[2]*shape[3]
X = tf.reshape(X, shape=[-1,full_size])
for i in range(c,len(self.channels)-1):
if i==c: n1=full_size
else: n1=self.channels[i]
n2=self.channels[i+1]
# W,b:full_connect
name_W='W_full'+str(i+1)
name_b='b_full'+str(i+1)
self.W.append(tf.Variable(tf.random_normal(shape=[n1,n2], stddev=0.1), name=name_W))
self.b.append(tf.Variable(tf.constant(0.1, shape=[n2]),name=name_b))
if self.dropout>0:
X = tf.nn.dropout(X, self.keep_prob) # Apply Dropout
# 全连接
"""transform < full_connect >"""
z=tf.add(tf.matmul(X,self.W[-1]), self.b[-1])
if i==len(self.channels)-2:
logist = z
pred = act_func(self.output_act_func)(z) # Relu activation
else:
X = act_func(self.hidden_act_func)(z) # Relu activation
return logist,pred
def __init__(
self,
hidden_act_func='relu',
output_act_func='softmax',
loss_func='mse', # gauss 激活函数会自动转换为 mse 损失函数
struct=[784, 100, 100, 10],
lr=1e-4,
momentum=0.5,
use_for='classification',
bp_algorithm='adam',
epochs=100,
batch_size=32,
dropout=0.3,
units_type=['gauss', 'bin'],
rbm_lr=1e-3,
rbm_epochs=30,
cd_k=1):
Model.__init__(self, 'DBN')
self.output_act_func = output_act_func
self.hidden_act_func = hidden_act_func
self.loss_func = loss_func
self.use_for = use_for
self.bp_algorithm = bp_algorithm
self.lr = lr
self.momentum = momentum
self.epochs = epochs
self.struct = struct
self.batch_size = batch_size
self.dropout = dropout
self.dbm_struct = struct[:-1]
self.units_type = units_type
self.cd_k = cd_k
self.rbm_lr = rbm_lr
self.rbm_epochs = rbm_epochs
# if rbm_v_type=='bin':
# self.hidden_act_func='sigmoid'
# elif rbm_v_type=='gauss':
# self.hidden_act_func='affine'
if output_act_func == 'gauss':
self.loss_func = 'mse'
self.hidden_act = act_func(self.hidden_act_func)
self.output_act = act_func(self.output_act_func)
self.build_model()
def conv2d(self, img, w, b):
"""tf.nn.conv2d
input = [batch, in_height, in_width, in_channels]
strides=[filter_height, filter_width, in_channels, out_channels]
"""
z = tf.nn.conv2d(img, w, strides=[1, 1, 1, 1], padding='VALID') + b
return act_func(self.hidden_act_func)(z)
def __init__(
self,
output_func='softmax',
hidden_func='affine', # encoder：[sigmoid] | [affine]
use_for='classification',
loss_func='mse', # decoder：[sigmoid] with ‘cross_entropy’ | [affine] with ‘mse’
struct=[784, 100, 100, 10],
lr=1e-4,
epochs=60,
batch_size=32,
dropout=0,
ae_type='dae', # ae | dae | sae
act_type=['sigmoid', 'affine'],
noise_type='mn', # Gaussian noise (gs) | Masking noise (mn)
beta=0.25, # 惩罚因子权重（KL项 | 非噪声样本项）
p=0.5, # DAE：样本该维作为噪声的概率 / SAE稀疏性参数：期望的隐层平均活跃度（在训练批次上取平均）
ae_lr=1e-3,
ae_epochs=20,
pre_train=True):
Model.__init__(self, 'sup_sAE')
self.output_func = output_func
self.hidden_func = hidden_func
self.use_for = use_for
self.loss_func = loss_func
self.struct = struct
self.lr = lr
self.epochs = epochs
self.dropout = dropout
self.pre_train = pre_train
self.un_ae_struct = struct[:-1]
self.ae_type = ae_type
self.act_type = act_type
self.noise_type = noise_type
self.beta = beta
self.p = p
self.ae_epochs = ae_epochs
self.ae_lr = ae_lr
self.batch_size = batch_size
self.hidden_act = act_func(self.hidden_func)
self.output_act = act_func(self.output_func)
self.build_model()
def __init__(self,
output_act_func='softmax',
loss_func='cross_entropy',
use_for='classification',
bp_algorithm='sgd',
dbn_lr=1e-3,
momentum=0.5,
dbn_epochs=100,
dbn_struct=[784, 100, 100, 10],
rbm_v_type='bin',
rbm_epochs=10,
batch_size=32,
cd_k=1,
rbm_lr=1e-3,
dropout=1):
self.output_act_func = output_act_func
self.loss_func = loss_func
self.use_for = use_for
self.bp_algorithm = bp_algorithm
self.dbn_lr = dbn_lr
self.momentum = momentum
self.dbn_epochs = dbn_epochs
self.dbn_struct = dbn_struct
self.dbm_struct = dbn_struct[:-1]
self.rbm_v_type = rbm_v_type
self.rbm_epochs = rbm_epochs
self.batch_size = batch_size
self.cd_k = cd_k
self.rbm_lr = rbm_lr
self.dropout = dropout
if rbm_v_type == 'bin':
self.hidden_act_func = 'sigmoid'
elif rbm_v_type == 'gauss':
self.hidden_act_func = 'affine'
if output_act_func == 'gauss':
self.loss_func = 'mse'
self.hidden_act = act_func(self.hidden_act_func)
self.output_act = act_func(self.output_act_func)
self.build_model()
def transform(self, data_x):
# 得到网络输出值
next_data = data_x # 这个next_data是tf变量
for i in range(len(self.parameter_list)):
W = self.parameter_list[i][0]
b = self.parameter_list[i][1]
if self.dropout > 0:
next_data = tf.nn.dropout(next_data, self.keep_prob)
z = tf.add(tf.matmul(next_data, W), b)
if i == len(self.parameter_list) - 1:
logits = z
output_act = act_func(self.output_act_func)
pred = output_act(logits)
else:
hidden_act = act_func(self.hidden_func, self.h_act_p)
self.h_act_p = np.mod(self.h_act_p + 1, len(self.hidden_func))
next_data = hidden_act(z)
return logits, pred
def transform(self, x):
add_in = tf.add(tf.matmul(x, self.W), self.by)
return act_func(self.en_func)(add_in)
def reconstruction(self, h):
logist = tf.matmul(h, tf.transpose(self.W)) + self.bz
recon = act_func(self.de_func)(logist)
return logist, recon
def transform(self, x):
x = tf.cast(x, dtype=tf.float32)
self.W = tf.cast(self.W, dtype=tf.float32)
self.bh = tf.cast(self.bh, dtype=tf.float32)
add_in = tf.matmul(x, self.W) + self.bh
return act_func(self.en_func)(add_in)
def transform(self, x):
add_in = tf.matmul(x, self.W) + self.bh
return act_func(self.en_func)(add_in)
def func(name):
if name == 'bin': return act_func('sigmoid')
elif name == 'gauss': return act_func('affine')
elif name == 'g2': return act_func('gauss')
def build_model(self):
# feed
self.input_data = tf.placeholder(tf.float32, [None, self.img_shape[0]*self.img_shape[1]]) # N等于batch_size（训练）或_num_examples（测试）
self.label_data = tf.placeholder(tf.float32, [None, self.channels[-1]]) # N等于batch_size（训练）或_num_examples（测试）
# reshape X
X = tf.reshape(self.input_data, shape=[-1,self.img_shape[0] , self.img_shape[1], self.channels[0]])
# conv,max_pool
self.W=list()
self.b=list()
c=0
p=0
for layer in self.layer_tp:
if layer=='C':
# W,b:conv
name_W='W_conv'+str(c+1)
name_b='b_conv'+str(c+1)
W_shape=[self.fsize[c][0],self.fsize[c][1],self.channels[c],self.channels[c+1]]
self.W.append(tf.Variable(tf.random_normal(shape=W_shape, stddev=0.1), name=name_W))
self.b.append(tf.Variable(tf.constant(0.1, shape=[self.channels[c+1]]),name=name_b))
# Tensorboard
Summaries.scalars_histogram('_'+name_W,self.W[-1])
Summaries.scalars_histogram('_'+name_b,self.b[-1])
# 卷积操作
"""transform < conv >"""
X = self.conv2d(X, self.W[-1], self.b[-1])
c=c+1
else:
k_shape=[1,self.ksize[p][0],self.ksize[p][1],1]
# 池化操作
"""transform < max_pool >"""
X = self.max_pool(X, ksize=k_shape)
p=p+1
# full_connect 全连接层
shape=X.get_shape().as_list()
full_size=shape[1]* shape[2]*shape[3]
X = tf.reshape(X, shape=[-1,full_size])
for i in range(c,len(self.channels)-1):
if i==c: n1=full_size
else: n1=self.channels[i]
n2=self.channels[i+1]
# W,b:full_connect
name_W='W_full'+str(i+1)
name_b='b_full'+str(i+1)
self.W.append(tf.Variable(tf.random_normal(shape=[n1,n2], stddev=0.1), name=name_W))
self.b.append(tf.Variable(tf.constant(0.1, shape=[n2]),name=name_b))
# Tensorboard
Summaries.scalars_histogram('_'+name_W,self.W[-1])
Summaries.scalars_histogram('_'+name_b,self.b[-1])
# 全连接
"""transform < full_connect >"""
z=tf.add(tf.matmul(X,self.W[-1]), self.b[-1])
if i==len(self.channels)-2:
self.pred = act_func(self.output_act_func)(z) # Relu activation
else:
X = act_func(self.hidden_act_func)(z) # Relu activation
if self.dropout<1:
X = tf.nn.dropout(X, self.dropout) # Apply Dropout
# loss,trainer
self.parameter_list=list()
for i in range(len(self.W)):
self.parameter_list.append(self.W[i])
self.parameter_list.append(self.b[i])
_loss=Loss(label_data=self.label_data,
pred=self.pred,
output_act_func=self.output_act_func)
self.loss=_loss.get_loss_func(self.loss_func)
self.train_batch_bp=tf.train.AdamOptimizer(learning_rate=self.cnn_lr).minimize(self.loss, var_list=self.parameter_list)
# accuracy
_ac=Accuracy(label_data=self.label_data,
pred=self.pred)
self.accuracy=_ac.accuracy()
# Tensorboard
tf.summary.scalar('loss',self.loss)
tf.summary.scalar('accuracy',self.accuracy)
self.merge = tf.summary.merge(tf.get_collection(tf.GraphKeys.SUMMARIES,tf.get_default_graph()._name_stack))
def conditional_probability(name):
if name == 'bin': return act_func('sigmoid')
elif name == 'gauss': return act_func('affine')
