data_dict = {'epoch': [], 'time': [], 'train_loss': [], 'train_acc': [], 'val_loss': [], 'val_acc': []}
# パラメータ利用, 全結合パラメータの凍結
new_net = parameter_use('./result3/pkl1/original_train_epoch50.pkl')
# 全結合層、畳み込み層のリスト
dense_list = [module for module in new_net.modules() if isinstance(module, nn.Linear)]
conv_list = [module for module in new_net.modules() if isinstance(module, nn.Conv2d)]
conv_count = len(conv_list)
# for dense in dense_list:
# dense.weight.requires_grad = False
optimizer = optim.SGD(new_net.parameters(), lr=0.01, momentum=0.9, weight_decay=5e-4)
# マスクのオブジェクト
ch_mask = [ChannelMaskGenerator() for _ in range(conv_count)]
de_mask = [DenseMaskGenerator() for _ in dense_list]
for i, dense in enumerate(dense_list):
de_mask[i].mask = np.where(np.abs(dense.weight.data.clone().cpu().detach().numpy()) == 0, 0, 1)
inv_prune_ratio = 10
# channel_pruning
for count in range(1, inv_prune_ratio + 9):
print(f'\nchannel pruning: {count}')
# ノルムの合計を保持
channel_l1norm_for_each_layer = [list() for _ in range(conv_count)]
# ノルムの取得, 昇順にソート
for i, conv in enumerate(conv_list):
channel_l1norm_for_each_layer[i] = [np.sum(torch.abs(param).cpu().detach().numpy()) for param in conv.weight]
channel_l1norm_for_each_layer[i].sort()
def train(self, gene, g_count, dense_num):
# 枝刈り後パラメータ利用
self.network = parameter_use('./result/CIFAR10_dense_conv_prune.pkl')
for param in self.network.features.parameters():
param.requires_grad = False
# 全結合層のリスト
dense_list = [
self.network.classifier[i]
for i in range(len(self.network.classifier))
if isinstance(self.network.classifier[i], nn.Linear)
]
# マスクのオブジェクト
de_mask = [DenseMaskGenerator() for _ in range(len(dense_list))]
for i, dense in enumerate(dense_list):
de_mask[i].mask = np.where(
np.abs(dense.weight.data.clone().cpu().detach().numpy()) == 0,
0, 1)
# 追加
with torch.no_grad():
for j in range(len(
dense_list[dense_num].weight.data.cpu().numpy())):
if np.all(de_mask[dense_num].mask[j, :] == 0) and np.all(
de_mask[dense_num + 1].mask[:, j] == 0):
de_mask[dense_num].mask[j, :], de_mask[dense_num +
1].mask[:, j] = 1, 1
dense_list[dense_num].weight.data[j, :] = torch.tensor(
gene[0], device=device, dtype=dtype)
dense_list[dense_num + 1].weight.data[:, j] = torch.tensor(
gene[1], device=device, dtype=dtype)
print(f'add_neuron_dense{dense_num + 1}')
break
# パラメータの割合
weight_ratio = [
np.count_nonzero(dense.weight.cpu().detach().numpy()) /
np.size(dense.weight.cpu().detach().numpy())
for dense in dense_list
]
# 枝刈り後のニューロン数
neuron_num_new = [
dense_list[i].in_features - de_mask[i].neuron_number(dense.weight)
for i, dense in enumerate(dense_list)
]
print(f'parent{g_count + 1}: ') if g_count < 2 else print(
f'children{g_count - 1}: ')
for i in range(len(dense_list)):
print(f'dense{i + 1}_param: {weight_ratio[i]:.4f}',
end=", " if i != len(dense_list) - 1 else "\n")
for i in range(len(dense_list)):
print(f'neuron_number{i + 1}: {neuron_num_new[i]}',
end=", " if i != len(dense_list) - 1 else "\n")
similarity = 0
# ニューロン間の類似度
for param1, param2 in zip(
dense_list[dense_num].weight.data.clone().cpu().numpy(),
torch.t(dense_list[dense_num +
1].weight.data.clone()).cpu().numpy()):
similarity += neuron_euclidean_distance(gene[0],
np.sum(np.abs(param1)))
similarity += neuron_euclidean_distance(gene[1],
np.sum(np.abs(param2)))
# similarity += cos_sim(gene[0], np.sum(np.abs(param1)))
# similarity += cos_sim(gene[1], np.sum(np.abs(param2)))
f_num_epochs = 1
eva = 0
for epoch in range(f_num_epochs):
# val
self.network.eval()
val_loss, val_acc = 0, 0
with torch.no_grad():
for images, labels in test_loader:
labels = labels.to(device)
outputs = self.network(images.to(device))
loss = criterion(outputs, labels)
val_loss += loss.item()
val_acc += (outputs.max(1)[1] == labels).sum().item()
avg_val_loss, avg_val_acc = val_loss / len(
test_loader.dataset), val_acc / len(test_loader.dataset)
eva = avg_val_loss
print(
f'epoch [{epoch + 1}/{f_num_epochs}], val_loss: {avg_val_loss:.4f}, val_acc: {avg_val_acc:.4f}'
)
# 結果の保存
input_data = [g_count, epoch + 1, avg_val_loss, avg_val_acc]
result_save('./result/result_add_neurons_not_train.csv', data_dict,
input_data)
return 10000 * eva + similarity / 100000
# 枝刈り後畳み込み層・全結合層・係数パラメータのリスト
conv_list = [
module for module in new_net.modules() if isinstance(module, nn.Conv2d)
]
dense_list = [
module for module in new_net.modules() if isinstance(module, nn.Linear)
]
param_list = [
module for module in new_net.modules() if isinstance(module, nn.Parameter)
]
# マスクのオブジェクト
ch_mask = [ChannelMaskGenerator() for _ in range(len(conv_list))]
for i, conv in enumerate(conv_list):
ch_mask[i].mask = np.where(
np.abs(conv.weight.data.clone().cpu().detach().numpy()) == 0, 0, 1)
de_mask = [DenseMaskGenerator() for _ in range(len(dense_list))]
for i, dense in enumerate(dense_list):
de_mask[i].mask = np.where(
np.abs(dense.weight.data.clone().cpu().detach().numpy()) == 0, 0, 1)
gen_num = 300
add_channel_num = 3
optimizer = optim.SGD(new_net.parameters(),
lr=0.01,
momentum=0.9,
weight_decay=5e-4)
# 追加前重み分布の描画
# for i in range(len(conv_list)):
# before_weight = [np.sum(conv_list[i].weight.data[k].cpu().detach().numpy()) for k
# in range(len(conv_list[i].weight.data.cpu().numpy()))]
def train(self, gene, g_count, conv_num):
# 枝刈り後パラメータ利用
self.network = parameter_use('./result/CIFAR10_dense_conv_prune.pkl')
for param in self.network.classifier.parameters():
param.requires_grad = False
# optimizer = optim.SGD(self.network.parameters(), lr=0.01, momentum=0.9, weight_decay=5e-4)
# 畳み込み層のリスト
conv_list = [self.network.features[i] for i in range(len(self.network.features)) if
isinstance(self.network.features[i], nn.Conv2d)]
# 全結合層のリスト
dense_list = [self.network.classifier[i] for i in range(len(self.network.classifier)) if
isinstance(self.network.classifier[i], nn.Linear)]
# 畳み込み層マスクのオブジェクト
ch_mask = [ChannelMaskGenerator() for _ in range(len(conv_list))]
for i, conv in enumerate(conv_list):
ch_mask[i].mask = np.where(np.abs(conv.weight.data.clone().cpu().detach().numpy()) == 0, 0, 1)
# 全結合層マスクのオブジェクト
de_mask = [DenseMaskGenerator() for _ in dense_list]
# 追加
with torch.no_grad():
for j in range(len(conv_list[conv_num].weight.data.cpu().numpy())):
if np.sum(np.abs(ch_mask[conv_num].mask[j])) < 25 * (self.count + 1) + 1:
ch_mask[conv_num].mask[j] = 1
conv_list[conv_num].weight.data[j] = torch.tensor(gene, device=device, dtype=dtype)
if conv_num != len(conv_list) - 1:
ch_mask[conv_num + 1].mask[j, :] = 1
conv_list[conv_num + 1].weight.data[:, j] = original_conv_list[conv_num + 1].weight.data[:, j].clone()
print(f'add_filter_conv{conv_num + 1}')
break
# パラメータの割合
weight_ratio = [np.count_nonzero(conv.weight.cpu().detach().numpy()) /
np.size(conv.weight.cpu().detach().numpy()) for conv in conv_list]
# 枝刈り後のチャネル数
channel_num_new = [conv_list[i].out_channels - ch_mask[i].channel_number(conv.weight) for i, conv in
enumerate(conv_list)]
print(f'parent{g_count + 1}: ') if g_count < 2 else print(f'children{g_count - 1}: ')
for i in range(len(conv_list)):
print(f'conv{i + 1}_param: {weight_ratio[i]:.4f}', end=", " if i != len(conv_list) - 1 else "\n")
for i in range(len(conv_list)):
print(f'channel_number{i + 1}: {channel_num_new[i]}', end=", " if i != len(conv_list) - 1 else "\n")
similarity = 0
# チャネル間の類似度
for i in range(conv_list[conv_num].out_channels):
similarity += channel_euclidean_distance(gene, conv_list[conv_num].weight.data.cpu().detach().numpy()[i])
# similarity += cos_sim(gene, conv_list[conv_num].weight.data.cpu().detach().numpy()[i])
f_num_epochs = 1
eva = 0
# for param in self.network.features.parameters():
# param.requires_grad = False
# for param in self.network.classifier.parameters():
# param.requires_grad = True
# finetune
for epoch in range(f_num_epochs):
# train
# self.network.train()
# train_loss, train_acc = 0, 0
# for i, (images, labels) in enumerate(train_loader):
# images, labels = images.to(device), labels.to(device)
# optimizer.zero_grad()
# outputs = self.network(images)
# loss = criterion(outputs, labels)
# train_loss += loss.item()
# train_acc += (outputs.max(1)[1] == labels).sum().item()
# loss.backward()
# optimizer.step()
# with torch.no_grad():
# for j, dense in enumerate(dense_list):
# if de_mask[j].mask is None:
# break
# dense.weight.data *= torch.tensor(de_mask[j].mask, device=device, dtype=dtype)
# avg_train_loss, avg_train_acc = train_loss / len(train_loader.dataset), train_acc / len(train_loader.dataset)
avg_train_loss, avg_train_acc = 0, 0
# val
self.network.eval()
val_loss, val_acc = 0, 0
with torch.no_grad():
for images, labels in test_loader:
labels = labels.to(device)
outputs = self.network(images.to(device))
loss = criterion(outputs, labels)
val_loss += loss.item()
val_acc += (outputs.max(1)[1] == labels).sum().item()
avg_val_loss, avg_val_acc = val_loss / len(test_loader.dataset), val_acc / len(test_loader.dataset)
eva = avg_val_loss
print(f'epoch [{epoch + 1}/{f_num_epochs}], train_loss: {avg_train_loss:.4f}'
f', train_acc: {avg_train_acc:.4f}, val_loss: {avg_val_loss:.4f}, val_acc: {avg_val_acc:.4f}')
# 結果の保存
input_data = [g_count, epoch + 1, avg_train_loss, avg_train_acc, avg_val_loss, avg_val_acc]
result_save('./result/result_add_channels_not_train.csv', data_dict, input_data)
return 10000 * eva + similarity