def init_local_model(self, device):
### Select the model
if self.params.t_model_version == 'resnet18':
self.m_local = (resnet.ResNet18()).to(device) if self.params.t_cuda else (resnet.ResNet18())
elif self.params.t_model_version == 'alexnet':
self.m_local = (alexnet.AlexNet()).to(device) if self.params.t_cuda else (alexnet.AlexNet())
elif self.params.t_model_version == 'cnn':
self.m_local = (cnn.CNN()).to(device) if self.params.t_cuda else (cnn.CNN())
def model_fn(mode, features, labels, params):
TRAIN = mode == tf.estimator.ModeKeys.TRAIN
EVAL = mode == tf.estimator.ModeKeys.EVAL
PREDICT = mode == tf.estimator.ModeKeys.PREDICT
if not TRAIN:
params['dropout_rate'] = 0.0
if params['model'] == 'CNN':
graph = cnn.CNN(mode, params)
elif params['model'] == 'LSTM':
graph = lstm.LSTM(mode, params)
elif params['model'] == 'BILSTM':
graph = bilstm.BILSTM(mode, params)
elif params['model'] == 'ATTBILSTM':
graph = attbilstm.ATTBILSTM(mode, params)
elif params['model'] == 'ATTLSTM':
graph = attlstm.ATTLSTM(mode, params)
elif params['model'] == 'BILSTMCNN':
graph = bilstmcnn.BILSTMCNN(mode, params)
else:
raise ValueError('Select a training model (CNN or LSTM)')
logits, predict = graph.build(features['inputs'], features['length'])
if PREDICT:
predictions = {'indices': predict, 'logits': logits}
return tf.estimator.EstimatorSpec(mode, predictions=predictions)
labels_ = tf.one_hot(labels, params['n_label'])
loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits_v2(logits=logits,
labels=labels_))
if params['model'] == ('ATTBILSTM' or 'ATTLSTM'):
penalty = graph.penalty
loss_P = tf.reduce_mean(penalty * params['p_coef'])
loss = loss + loss_P
accuracy = tf.metrics.accuracy(labels=labels, predictions=predict)
metrics = {'accuracy': accuracy}
tf.summary.scalar('accuracy_train', accuracy[1])
if EVAL:
return tf.estimator.EstimatorSpec(mode,
loss=loss,
eval_metric_ops=metrics)
assert TRAIN, 'Select a mode'
optimizer = tf.train.AdamOptimizer(params['learning_rate'])
train_op = optimizer.minimize(loss, global_step=tf.train.get_global_step())
return tf.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op)
def __init__(self,
opt,
x_dim,
h_dim1,
h_dim2,
z_dim,
mgn_model_type="cnn_2layers"):
super(MGN, self).__init__()
for key, val in opt.__dict__.items():
setattr(self, key, val)
# encoder part
self.mgn_model_type = mgn_model_type
self.ch = self.image_shape[0]
self.imageNet_shape = (224, 224)
# layer_size = 64
if self.mgn_model_type == "cnn_2layers":
self.cnn = cnn.CNN(in_channel=self.image_shape[0],
img_sz=self.img_sz)
# self.layer1 = nn.Sequential(
# nn.Conv2d(self.ch, layer_size, kernel_size=5, stride=1, padding=2),
# nn.ReLU(),
# nn.MaxPool2d(kernel_size=2, stride=2))
# self.layer2 = nn.Sequential(
# nn.Conv2d(layer_size, layer_size, kernel_size=5, stride=1, padding=2),
# nn.ReLU(),
# nn.MaxPool2d(kernel_size=2, stride=2))
# self.fc1 = nn.Linear(8 * 8 * 16, h_dim1)
# self.drop_out = nn.Dropout()
# self.fc2 = nn.Linear(1000, 10)
last_cnn_dim = 8 * 8 * self.cnn.layer_size
elif self.mgn_model_type == self.pretrained_model_type:
img_sz = (3, 32, 32)
print("Loading pretrained ImageNet model {} ...".format(
self.pretrained_model_type))
self.pretrained = pretrained.PRETRAINED(
pretrained_model_type=self.pretrained_model_type,
in_channel=self.image_shape[0],
img_sz=self.img_sz)
# self.image_size_change = nn.AdaptiveAvgPool2d((224, 224))
### This upscales the image
# self.adap_img = nn.AdaptiveAvgPool2d(self.imageNet_shape)
last_cnn_dim = 1000
elif self.mgn_model_type == 'mlp':
self.net1 = nn.Linear(x_dim, h_dim1)
last_cnn_dim = h_dim1
else:
raise ValueError('No such MGN model type such as {}'.format(
self.mgn_model_type))
self.fc31 = nn.Linear(last_cnn_dim, z_dim)
self.fc32 = nn.Linear(last_cnn_dim, z_dim)
def by_name(name, FLAGS, training=False):
name = name.lower()
if name == 'cnn':
print('train polynomial activations?', FLAGS.train_poly_act)
print("batch norm?", FLAGS.batch_norm)
model = cnn.CNN(training=training,
train_poly_act=FLAGS.train_poly_act,
batch_norm=FLAGS.batch_norm)
else:
raise ValueError('No such model %s' % name)
return model
download=False) # 为True需要先下载，然后处理；为False使用下载完成的
test_dataset = datasets.MNIST(root='./MNIST_data',
train=False,
transform=data_tf)
train_loader = DataLoader(train_dataset, batch_size=batch_size,
shuffle=True) #训练集，shuffle为True时打乱数据，让数据更有选择随机性
test_loader = DataLoader(test_dataset, batch_size=batch_size,
shuffle=False) #对测试集进行迭代器编号
# 60000张训练集，batch_size为50 => 60000/50=1200,即每次模型输入50个数据，要进行1200次
# print(len(train_loader))
# 选择模型
model = cnn.CNN()
#model = lenet.LeNet()
#print(model) # 查看模型架构
criterion = nn.CrossEntropyLoss() # 定义损失函数
optimizer = optim.SGD(model.parameters(), lr=learning_rate) # 定义优化器
#'''训练之前定义一些空列表来添加训练得到的一些值 '''
train_losses = []
train_acces = []
eval_losses = []
eval_acces = []
for epoch in range(num_epoches):
# 每次epoch训练
train_loss = 0
train_acc = 0
def load_model(model, dataset, actfun, k, p, g, num_params, perm_method, device, resnet_ver, resnet_width, verbose):
model_params = []
if dataset == 'mnist' or dataset == 'fashion_mnist':
input_channels, input_dim, output_dim = 1, 28, 10
elif dataset == 'cifar10' or dataset == 'svhn':
input_channels, input_dim, output_dim = 3, 32, 10
elif dataset == 'cifar100':
input_channels, input_dim, output_dim = 3, 32, 100
if model == 'nn' or model == 'mlp':
if dataset == 'mnist' or dataset == 'fashion_mnist':
input_dim = 784
elif dataset == 'cifar10' or dataset == 'svhn':
input_dim = 3072
elif dataset == 'cifar100':
input_dim = 3072
elif dataset == 'iris':
input_dim, output_dim = 4, 3
model = mlp.MLP(actfun=actfun,
input_dim=input_dim,
output_dim=output_dim,
k=k,
p=p,
g=g,
num_params=num_params,
permute_type=perm_method).to(device)
model_params.append({'params': model.batch_norms.parameters(), 'weight_decay': 0})
model_params.append({'params': model.linear_layers.parameters()})
if actfun == 'combinact':
model_params.append({'params': model.all_alpha_primes.parameters(), 'weight_decay': 0})
elif model == 'cnn':
model = cnn.CNN(actfun=actfun,
num_input_channels=input_channels,
input_dim=input_dim,
num_outputs=output_dim,
k=k,
p=p,
g=g,
num_params=num_params,
permute_type=perm_method).to(device)
model_params.append({'params': model.conv_layers.parameters()})
model_params.append({'params': model.pooling.parameters()})
model_params.append({'params': model.batch_norms.parameters(), 'weight_decay': 0})
model_params.append({'params': model.linear_layers.parameters()})
if actfun == 'combinact':
model_params.append({'params': model.all_alpha_primes.parameters(), 'weight_decay': 0})
elif model == 'resnet':
model = preact_resnet.PreActResNet(resnet_ver=resnet_ver,
actfun=actfun,
in_channels=input_channels,
out_channels=output_dim,
k=k,
p=p,
g=g,
permute_type=perm_method,
width=resnet_width,
verbose=verbose).to(device)
model_params = model.parameters()
return model, model_params
def run_cnn_model(self):
config = cnn.Config()
cnn_model = cnn.CNN(config)
with tf.Session() as sess:
cnn_model.run_epoch(sess, self.X_train, self.y_train, self.X_test, self.y_test)
# preprocessing
X = X * 256
Xtest = Xtest * 256
y = y.reshape(len(y), 1)
ytest = ytest.reshape(len(ytest), 1)
X -= int(np.mean(X))
X /= int(np.std(X))
# print("Fitting pricipal components")
# model = pca.AlternativePCA(k=100)
# model.fit(X)
# print("Compressing")
# Z = model.compress(X)
# print("Expanding")
# Xhat_pca = model.expand(Z)
start_time = time.time()
model = cnn.CNN(batch_size=64, num_epochs=4)
model.fit(X, y)
y_pred = model.predict(X)
tr_error = np.mean(y_pred != y)
y_pred = model.predict(Xtest)
te_error = np.mean(y_pred != ytest)
print("Training error = %.4f" % tr_error)
print("Testing error = %.4f" % te_error)
print("Runtime: %s seconds" % (time.time() - start_time))
else:
pass