def __init__(self, output_act_func='softmax', hidden_act_func='relu', loss_func='cross_entropy', use_for='classification', bp_algorithm='adam', lr=1e-3, momentum=0.9, epochs=100, width=5, struct=[784, 100, 10], batch_size=32, dropout=0): Model.__init__(self, 'RNN') self.output_act_func = output_act_func self.hidden_act_func = hidden_act_func self.loss_func = loss_func self.use_for = use_for self.bp_algorithm = bp_algorithm self.lr = lr self.momentum = momentum self.epochs = epochs self.width = width self.struct = struct self.batch_size = batch_size self.dropout = dropout if output_act_func == 'gauss': self.loss_func = 'mse' self.hidden_act = act_func(self.hidden_act_func) self.output_act = act_func(self.output_act_func) self.build_model()
def __init__(self, out_func='softmax', en_func='sigmoid', # encoder:[sigmoid] use_for='classification', loss_func='mse', # decoder:[sigmoid] with ‘cross_entropy’ | [relu] with ‘mse’ ae_type='ae', # ae | dae | sae noise_type='gs', # Gaussian noise (gs) | Masking noise (mn) beta=0.5, # 惩罚因子权重(第二项损失的系数) p=0.5, # DAE:样本该维作为噪声的概率 / SAE稀疏性参数:期望的隐层平均活跃度(在训练批次上取平均) sup_ae_struct=[784, 100, 100, 10], sup_ae_epochs=100, ae_epochs=10, batch_size=32, ae_lr=1e-3, dropout=1): self.out_func=out_func self.en_func=en_func self.use_for=use_for self.loss_func=loss_func self.ae_type = ae_type self.noise_type = noise_type self.beta = beta self.p = p self.sup_ae_struct = sup_ae_struct self.un_ae_struct = sup_ae_struct[:-1] self.sup_ae_epochs=sup_ae_epochs self.ae_epochs = ae_epochs self.batch_size = batch_size self.ae_lr = ae_lr self.dropout = dropout self.hidden_act=act_func(self.en_func) self.output_act=act_func(self.out_func) self.build_model()
def transform(self,X): # conv,max_pool self.W=list() self.b=list() c=0 p=0 for layer in self.layer_tp: if layer=='C': # W,b:conv name_W='W_conv'+str(c+1) name_b='b_conv'+str(c+1) W_shape=[self.fsize[c][0],self.fsize[c][1],self.channels[c],self.channels[c+1]] self.W.append(tf.Variable(tf.random_normal(shape=W_shape, stddev=0.1), name=name_W)) self.b.append(tf.Variable(tf.constant(0.1, shape=[self.channels[c+1]]),name=name_b)) # 卷积操作 """transform < conv >""" X = self.conv2d(X, self.W[-1], self.b[-1]) c=c+1 else: k_shape=[1,self.ksize[p][0],self.ksize[p][1],1] # 池化操作 """transform < max_pool >""" X = self.max_pool(X, ksize=k_shape) p=p+1 # full_connect 全连接层 shape=X.get_shape().as_list() full_size=shape[1]* shape[2]*shape[3] X = tf.reshape(X, shape=[-1,full_size]) for i in range(c,len(self.channels)-1): if i==c: n1=full_size else: n1=self.channels[i] n2=self.channels[i+1] # W,b:full_connect name_W='W_full'+str(i+1) name_b='b_full'+str(i+1) self.W.append(tf.Variable(tf.random_normal(shape=[n1,n2], stddev=0.1), name=name_W)) self.b.append(tf.Variable(tf.constant(0.1, shape=[n2]),name=name_b)) if self.dropout>0: X = tf.nn.dropout(X, self.keep_prob) # Apply Dropout # 全连接 """transform < full_connect >""" z=tf.add(tf.matmul(X,self.W[-1]), self.b[-1]) if i==len(self.channels)-2: logist = z pred = act_func(self.output_act_func)(z) # Relu activation else: X = act_func(self.hidden_act_func)(z) # Relu activation return logist,pred
def __init__( self, hidden_act_func='relu', output_act_func='softmax', loss_func='mse', # gauss 激活函数会自动转换为 mse 损失函数 struct=[784, 100, 100, 10], lr=1e-4, momentum=0.5, use_for='classification', bp_algorithm='adam', epochs=100, batch_size=32, dropout=0.3, units_type=['gauss', 'bin'], rbm_lr=1e-3, rbm_epochs=30, cd_k=1): Model.__init__(self, 'DBN') self.output_act_func = output_act_func self.hidden_act_func = hidden_act_func self.loss_func = loss_func self.use_for = use_for self.bp_algorithm = bp_algorithm self.lr = lr self.momentum = momentum self.epochs = epochs self.struct = struct self.batch_size = batch_size self.dropout = dropout self.dbm_struct = struct[:-1] self.units_type = units_type self.cd_k = cd_k self.rbm_lr = rbm_lr self.rbm_epochs = rbm_epochs # if rbm_v_type=='bin': # self.hidden_act_func='sigmoid' # elif rbm_v_type=='gauss': # self.hidden_act_func='affine' if output_act_func == 'gauss': self.loss_func = 'mse' self.hidden_act = act_func(self.hidden_act_func) self.output_act = act_func(self.output_act_func) self.build_model()
def conv2d(self, img, w, b): """tf.nn.conv2d input = [batch, in_height, in_width, in_channels] strides=[filter_height, filter_width, in_channels, out_channels] """ z = tf.nn.conv2d(img, w, strides=[1, 1, 1, 1], padding='VALID') + b return act_func(self.hidden_act_func)(z)
def __init__( self, output_func='softmax', hidden_func='affine', # encoder:[sigmoid] | [affine] use_for='classification', loss_func='mse', # decoder:[sigmoid] with ‘cross_entropy’ | [affine] with ‘mse’ struct=[784, 100, 100, 10], lr=1e-4, epochs=60, batch_size=32, dropout=0, ae_type='dae', # ae | dae | sae act_type=['sigmoid', 'affine'], noise_type='mn', # Gaussian noise (gs) | Masking noise (mn) beta=0.25, # 惩罚因子权重(KL项 | 非噪声样本项) p=0.5, # DAE:样本该维作为噪声的概率 / SAE稀疏性参数:期望的隐层平均活跃度(在训练批次上取平均) ae_lr=1e-3, ae_epochs=20, pre_train=True): Model.__init__(self, 'sup_sAE') self.output_func = output_func self.hidden_func = hidden_func self.use_for = use_for self.loss_func = loss_func self.struct = struct self.lr = lr self.epochs = epochs self.dropout = dropout self.pre_train = pre_train self.un_ae_struct = struct[:-1] self.ae_type = ae_type self.act_type = act_type self.noise_type = noise_type self.beta = beta self.p = p self.ae_epochs = ae_epochs self.ae_lr = ae_lr self.batch_size = batch_size self.hidden_act = act_func(self.hidden_func) self.output_act = act_func(self.output_func) self.build_model()
def __init__(self, output_act_func='softmax', loss_func='cross_entropy', use_for='classification', bp_algorithm='sgd', dbn_lr=1e-3, momentum=0.5, dbn_epochs=100, dbn_struct=[784, 100, 100, 10], rbm_v_type='bin', rbm_epochs=10, batch_size=32, cd_k=1, rbm_lr=1e-3, dropout=1): self.output_act_func = output_act_func self.loss_func = loss_func self.use_for = use_for self.bp_algorithm = bp_algorithm self.dbn_lr = dbn_lr self.momentum = momentum self.dbn_epochs = dbn_epochs self.dbn_struct = dbn_struct self.dbm_struct = dbn_struct[:-1] self.rbm_v_type = rbm_v_type self.rbm_epochs = rbm_epochs self.batch_size = batch_size self.cd_k = cd_k self.rbm_lr = rbm_lr self.dropout = dropout if rbm_v_type == 'bin': self.hidden_act_func = 'sigmoid' elif rbm_v_type == 'gauss': self.hidden_act_func = 'affine' if output_act_func == 'gauss': self.loss_func = 'mse' self.hidden_act = act_func(self.hidden_act_func) self.output_act = act_func(self.output_act_func) self.build_model()
def transform(self, data_x): # 得到网络输出值 next_data = data_x # 这个next_data是tf变量 for i in range(len(self.parameter_list)): W = self.parameter_list[i][0] b = self.parameter_list[i][1] if self.dropout > 0: next_data = tf.nn.dropout(next_data, self.keep_prob) z = tf.add(tf.matmul(next_data, W), b) if i == len(self.parameter_list) - 1: logits = z output_act = act_func(self.output_act_func) pred = output_act(logits) else: hidden_act = act_func(self.hidden_func, self.h_act_p) self.h_act_p = np.mod(self.h_act_p + 1, len(self.hidden_func)) next_data = hidden_act(z) return logits, pred
def transform(self, x): add_in = tf.add(tf.matmul(x, self.W), self.by) return act_func(self.en_func)(add_in)
def reconstruction(self, h): logist = tf.matmul(h, tf.transpose(self.W)) + self.bz recon = act_func(self.de_func)(logist) return logist, recon
def transform(self, x): x = tf.cast(x, dtype=tf.float32) self.W = tf.cast(self.W, dtype=tf.float32) self.bh = tf.cast(self.bh, dtype=tf.float32) add_in = tf.matmul(x, self.W) + self.bh return act_func(self.en_func)(add_in)
def transform(self, x): add_in = tf.matmul(x, self.W) + self.bh return act_func(self.en_func)(add_in)
def func(name): if name == 'bin': return act_func('sigmoid') elif name == 'gauss': return act_func('affine') elif name == 'g2': return act_func('gauss')
def build_model(self): # feed self.input_data = tf.placeholder(tf.float32, [None, self.img_shape[0]*self.img_shape[1]]) # N等于batch_size(训练)或_num_examples(测试) self.label_data = tf.placeholder(tf.float32, [None, self.channels[-1]]) # N等于batch_size(训练)或_num_examples(测试) # reshape X X = tf.reshape(self.input_data, shape=[-1,self.img_shape[0] , self.img_shape[1], self.channels[0]]) # conv,max_pool self.W=list() self.b=list() c=0 p=0 for layer in self.layer_tp: if layer=='C': # W,b:conv name_W='W_conv'+str(c+1) name_b='b_conv'+str(c+1) W_shape=[self.fsize[c][0],self.fsize[c][1],self.channels[c],self.channels[c+1]] self.W.append(tf.Variable(tf.random_normal(shape=W_shape, stddev=0.1), name=name_W)) self.b.append(tf.Variable(tf.constant(0.1, shape=[self.channels[c+1]]),name=name_b)) # Tensorboard Summaries.scalars_histogram('_'+name_W,self.W[-1]) Summaries.scalars_histogram('_'+name_b,self.b[-1]) # 卷积操作 """transform < conv >""" X = self.conv2d(X, self.W[-1], self.b[-1]) c=c+1 else: k_shape=[1,self.ksize[p][0],self.ksize[p][1],1] # 池化操作 """transform < max_pool >""" X = self.max_pool(X, ksize=k_shape) p=p+1 # full_connect 全连接层 shape=X.get_shape().as_list() full_size=shape[1]* shape[2]*shape[3] X = tf.reshape(X, shape=[-1,full_size]) for i in range(c,len(self.channels)-1): if i==c: n1=full_size else: n1=self.channels[i] n2=self.channels[i+1] # W,b:full_connect name_W='W_full'+str(i+1) name_b='b_full'+str(i+1) self.W.append(tf.Variable(tf.random_normal(shape=[n1,n2], stddev=0.1), name=name_W)) self.b.append(tf.Variable(tf.constant(0.1, shape=[n2]),name=name_b)) # Tensorboard Summaries.scalars_histogram('_'+name_W,self.W[-1]) Summaries.scalars_histogram('_'+name_b,self.b[-1]) # 全连接 """transform < full_connect >""" z=tf.add(tf.matmul(X,self.W[-1]), self.b[-1]) if i==len(self.channels)-2: self.pred = act_func(self.output_act_func)(z) # Relu activation else: X = act_func(self.hidden_act_func)(z) # Relu activation if self.dropout<1: X = tf.nn.dropout(X, self.dropout) # Apply Dropout # loss,trainer self.parameter_list=list() for i in range(len(self.W)): self.parameter_list.append(self.W[i]) self.parameter_list.append(self.b[i]) _loss=Loss(label_data=self.label_data, pred=self.pred, output_act_func=self.output_act_func) self.loss=_loss.get_loss_func(self.loss_func) self.train_batch_bp=tf.train.AdamOptimizer(learning_rate=self.cnn_lr).minimize(self.loss, var_list=self.parameter_list) # accuracy _ac=Accuracy(label_data=self.label_data, pred=self.pred) self.accuracy=_ac.accuracy() # Tensorboard tf.summary.scalar('loss',self.loss) tf.summary.scalar('accuracy',self.accuracy) self.merge = tf.summary.merge(tf.get_collection(tf.GraphKeys.SUMMARIES,tf.get_default_graph()._name_stack))
def conditional_probability(name): if name == 'bin': return act_func('sigmoid') elif name == 'gauss': return act_func('affine')