def __init__(self, pathModel, nnArchitecture, nnClassCount, transCrop):
#---- Initialize the network
if nnArchitecture == 'DENSE-NET-121': model = densenet121(False).cuda()
elif nnArchitecture == 'DENSE-NET-169':
model = densenet169(False).cuda()
elif nnArchitecture == 'DENSE-NET-201':
model = densenet201(False).cuda()
model = torch.nn.DataParallel(model).cuda()
modelCheckpoint = torch.load(pathModel)
model.load_state_dict(modelCheckpoint['best_model_wts'], strict=False)
self.model = model.module.features
self.model.eval()
#---- Initialize the weights
self.weights = list(self.model.parameters())[-2]
#---- Initialize the image transform - resize + normalize
normalize = transforms.Normalize([0.485, 0.456, 0.406],
[0.229, 0.224, 0.225])
transformList = []
transformList.append(transforms.Resize(transCrop))
transformList.append(transforms.ToTensor())
transformList.append(normalize)
self.transformSequence = transforms.Compose(transformList)
def go_test():
print("=" * 30)
model = densenet.densenet201(pretrained=True)
# Replace classification layer
model.classifier = torch.nn.Linear(model.classifier.in_features,
num_classes)
model.to(device)
# Load best performing model based on validation score
model.load_state_dict(
torch.load(f"snapshots/densenet201_best_{MODEL_NAME}.pth"))
print("[-] Performing validation...")
train_loader, val_loader, val_loader2 = get_train_val_split(batch_size)
with torch.no_grad():
val_accuracy = validate(model, val_loader, -1)
print("[*] Validation accuracy: {}".format(val_accuracy))
val_accuracy2 = validate(model, val_loader2, -1)
print("[*] Validation2 accuracy: {}\n".format(val_accuracy2))
print("[-] Performing testing...")
test_loader = get_test(batch_size)
with torch.no_grad():
test(model, test_loader)
def main():
##################
# Initialization #
##################
# Load a model pretrained on ImageNet
ensemble_model = ensemble.ensemble_ver3()
ensemble_model.to(device)
trained_models = [densenet.densenet201(pretrained=True) for i in range(5)]
# Replace classification layer
for i, model in enumerate(trained_models):
model.classifier = torch.nn.Linear(model.classifier.in_features, num_classes)
model.to(device)
# Load best performing model based on validation score
model.load_state_dict(torch.load(f"snapshots/densenet201_best_{MODEL_NAMES[i]}.pth"))
criterion = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=base_lr)
############
# Training #
############
train_loader, val_loader, val_loader2 = get_train_val_split(batch_size)
# Use model that performs best on validation for testing
best_val_accuracy = 0
for epoch in range(num_epochs):
# Train
print("="*30)
train(ensemble_model, trained_models, train_loader, optimizer, criterion, epoch)
# Validate
val_accuracy = validate(ensemble_model, trained_models, val_loader, epoch)
print("[*] Validation accuracy: {}".format(val_accuracy))
val_accuracy2 = validate(ensemble_model, trained_models, val_loader2, epoch)
print("[*] Validation2 accuracy: {}\n".format(val_accuracy2))
# New best performing model
if val_accuracy2 > best_val_accuracy:
best_val_accuracy = val_accuracy2
print("[*] New best accuracy!\n")
torch.save(ensemble_model.state_dict(), f"snapshots/densenet201_experiment_usual.pth")
###########
# Testing #
###########
print("="*30)
print("[-] Performing testing...")
# Load best performing model based on validation score
ensemble_model.load_state_dict(torch.load(f"snapshots/densenet201_experiment_usual.pth"))
test_loader = get_test(batch_size)
test(ensemble_model, trained_models, test_loader)
def go_test():
print("="*30)
ensemble_model = ensemble.ensemble_ver3()
ensemble_model.to(device)
ensemble_model.load_state_dict(torch.load(f"snapshots/densenet201_experiment_usual.pth"))
trained_models = [densenet.densenet201(pretrained=True) for i in range(5)]
for i, model in enumerate(trained_models):
model.classifier = torch.nn.Linear(model.classifier.in_features, num_classes)
model.to(device)
# Load best performing model based on validation score
model.load_state_dict(torch.load(f"snapshots/densenet201_best_{MODEL_NAMES[i]}.pth"))
print("[-] Performing testing...")
test_loader = get_test(batch_size)
test(ensemble_model, trained_models, test_loader)
def CreatNet(name):
if name == 'LSTM':
return LSTM(100, 27 * 5, 5)
elif name == 'CNN':
return CNN()
elif name == 'resnet18_1d':
return resnet18_1d()
elif name == 'dfcnn':
return dfcnn()
elif name in ['resnet101', 'resnet50', 'resnet18']:
if name == 'resnet101':
net = torchvision.models.resnet101(pretrained=False)
net.fc = nn.Linear(2048, 5)
elif name == 'resnet50':
net = torchvision.models.resnet50(pretrained=False)
net.fc = nn.Linear(2048, 5)
elif name == 'resnet18':
net = torchvision.models.resnet18(pretrained=False)
net.fc = nn.Linear(512, 5)
net.conv1 = nn.Conv2d(1, 64, 7, 2, 3, bias=False)
return net
elif 'densenet' in name:
if name == 'densenet121':
net = densenet.densenet121(pretrained=False, num_classes=5)
elif name == 'densenet201':
net = densenet.densenet201(pretrained=False, num_classes=5)
# net.features = nn.Sequential(OrderedDict([
# ('conv0', nn.Conv2d(1, 64, kernel_size=7, stride=2, padding=3, bias=False)),
# ('norm0', nn.BatchNorm2d(64)),
# ('relu0', nn.ReLU(inplace=True)),
# ('pool0', nn.MaxPool2d(kernel_size=3, stride=2, padding=1)),
# ]))
# net.classifier = nn.Linear(64, 5)
return net
def te():
retrain_log = "-" * 20 + "retrain" + "-" * 20 + '\n'
batch_size = 64
epoch = 200
with tf.Session() as sess:
data_x = tf.placeholder(tf.float32, [64, 32, 32, 3], name='input')
labels = tf.placeholder(tf.int32, [64, 100], name="label")
train_flag = tf.placeholder(tf.bool, name='train_flag')
# for block in pre_block:
# graph = block.graph + [[]]
# cell_list = []
# for cell in block.cell_list:
# if cell.type == 'conv':
# cell_list.append(
# Cell(cell.type, cell.filter_size * 2, cell.kernel_size, cell.activation))
# else:
# cell_list.append(cell)
# cell_list.append(Cell('pooling', 'max', 1))
# # repeat search
# for _ in range(NAS_CONFIG['nas_main']['repeat_search'] - 1):
# retrain_log = retrain_log + str(graph) + str(cell_list) + '\n'
# logits = self._inference(logits, graph, cell_list, train_flag)
# self.block_num += 1
# # add pooling layer only in last repeat block
# cell_list[-1] = Cell('pooling', 'max', 2)
# retrain_log = retrain_log + str(graph) + str(cell_list) + '\n'
logits = densenet201(data_x,
reuse=False,
is_training=True,
kernel_initializer=tf.orthogonal_initializer())
# logits = tf.nn.dropout(logits, keep_prob=1.0)
# softmax
# logits = self._makedense(logits, ('', [256, self.NUM_CLASSES], 'relu'))
retrain = True
global_step = tf.Variable(0, trainable=False, name='global_step')
accuracy = cal_accuracy(logits, labels)
loss = loss_(labels, logits)
train_op = train_op_(global_step, loss)
saver = tf.train.Saver(tf.global_variables())
sess.run(tf.global_variables_initializer())
train_data, train_label, test_data, test_label = inputs()
max_steps = (50000) // batch_size
num_iter = len(test_label) // batch_size
log = ''
run_metadata = tf.RunMetadata()
cost_time = 0
precision = np.zeros([epoch])
for ep in range(epoch):
# print("epoch", ep, ":")
# train step
start_time = time.time()
for step in range(max_steps):
batch_x = train_data[step * batch_size:(step + 1) * batch_size]
batch_y = train_label[step * batch_size:(step + 1) *
batch_size]
batch_x = DataSet().process(batch_x)
_, loss_value, acc = sess.run(
[train_op, loss, accuracy],
feed_dict={
data_x: batch_x,
labels: batch_y,
train_flag: True
},
options=tf.RunOptions(
trace_level=tf.RunOptions.FULL_TRACE),
run_metadata=run_metadata)
if np.isnan(loss_value):
return -1, saver, log
sys.stdout.write("\r>> train %d/%d loss %.4f acc %.4f" %
(step, max_steps, loss_value, acc))
sys.stdout.write("\n")
trace = timeline.Timeline(step_stats=run_metadata.step_stats)
trace_file = open('timeline.ctf.json', 'w')
trace_file.write(trace.generate_chrome_trace_format())
# evaluation step
for step in range(num_iter):
batch_x = test_data[step * batch_size:(step + 1) * batch_size]
batch_y = test_label[step * batch_size:(step + 1) * batch_size]
l, acc_ = sess.run([loss, accuracy],
feed_dict={
data_x: batch_x,
labels: batch_y,
train_flag: False
})
precision[ep] += acc_ / num_iter
sys.stdout.write("\r>> valid %d/%d loss %.4f acc %.4f" %
(step, num_iter, l, acc_))
sys.stdout.write("\n")
cost_time += (float(time.time() - start_time)) / 200
log += 'epoch %d: precision = %.3f, cost time %.3f\n' % (
ep, precision[ep], float(time.time() - start_time))
retrain_log += log
# NAS_LOG << ('eva', retrain_log)
return float(precision)
def retrain(self, pre_block):
tf.reset_default_graph()
assert self.train_num >= self.batch_size
self.block_num = len(
pre_block) * NAS_CONFIG['nas_main']['repeat_search'] + 1
retrain_log = "-" * 20 + "retrain" + "-" * 20 + '\n'
with tf.Session() as sess:
data_x, labels, logits, train_flag = self._get_input(sess, [])
# for block in pre_block:
# graph = block.graph + [[]]
# cell_list = []
# for cell in block.cell_list:
# if cell.type == 'conv':
# cell_list.append(
# Cell(cell.type, cell.filter_size * 2, cell.kernel_size, cell.activation))
# else:
# cell_list.append(cell)
# cell_list.append(Cell('pooling', 'max', 1))
# # repeat search
# for _ in range(NAS_CONFIG['nas_main']['repeat_search'] - 1):
# retrain_log = retrain_log + str(graph) + str(cell_list) + '\n'
# logits = self._inference(logits, graph, cell_list, train_flag)
# self.block_num += 1
# # add pooling layer only in last repeat block
# cell_list[-1] = Cell('pooling', 'max', 2)
# retrain_log = retrain_log + str(graph) + str(cell_list) + '\n'
logits = densenet201(
data_x,
reuse=False,
is_training=True,
kernel_initializer=tf.orthogonal_initializer())
# logits = tf.nn.dropout(logits, keep_prob=1.0)
# softmax
# logits = self._makedense(logits, ('', [256, self.NUM_CLASSES], 'relu'))
retrain = True
global_step = tf.Variable(0,
trainable=False,
name='global_step' + str(self.block_num))
accuracy = self._cal_accuracy(logits, labels)
loss = self._loss(labels, logits)
train_op = self._train_op(global_step, loss)
saver = tf.train.Saver(tf.global_variables())
sess.run(tf.global_variables_initializer())
if retrain:
self.train_data = np.concatenate(
(np.array(self.train_data), np.array(self.valid_data)),
axis=0).tolist()
self.train_label = np.concatenate(
(np.array(self.train_label), np.array(self.valid_label)),
axis=0).tolist()
max_steps = (self.NUM_EXAMPLES_FOR_TRAIN +
self.NUM_EXAMPLES_FOR_EVAL) // self.batch_size
test_data = copy.deepcopy(self.test_data)
test_label = copy.deepcopy(self.test_label)
num_iter = len(test_label) // self.batch_size
else:
max_steps = self.train_num // self.batch_size
test_data = copy.deepcopy(self.valid_data)
test_label = copy.deepcopy(self.valid_label)
num_iter = self.NUM_EXAMPLES_FOR_EVAL // self.batch_size
log = ''
run_metadata = tf.RunMetadata()
cost_time = 0
precision = np.zeros([self.epoch])
for ep in range(self.epoch):
# print("epoch", ep, ":")
# train step
start_time = time.time()
for step in range(max_steps):
batch_x = self.train_data[step *
self.batch_size:(step + 1) *
self.batch_size]
batch_y = self.train_label[step *
self.batch_size:(step + 1) *
self.batch_size]
batch_x = DataSet().process(batch_x)
_, loss_value, acc = sess.run(
[train_op, loss, accuracy],
feed_dict={
data_x: batch_x,
labels: batch_y,
train_flag: True
},
options=tf.RunOptions(
trace_level=tf.RunOptions.FULL_TRACE),
run_metadata=run_metadata)
if np.isnan(loss_value):
return -1, saver, log
sys.stdout.write("\r>> train %d/%d loss %.4f acc %.4f" %
(step, max_steps, loss_value, acc))
sys.stdout.write("\n")
trace = timeline.Timeline(step_stats=run_metadata.step_stats)
trace_file = open('timeline.ctf.json', 'w')
trace_file.write(trace.generate_chrome_trace_format())
# evaluation step
for step in range(num_iter):
batch_x = test_data[step * self.batch_size:(step + 1) *
self.batch_size]
batch_y = test_label[step * self.batch_size:(step + 1) *
self.batch_size]
l, acc_ = sess.run([loss, accuracy],
feed_dict={
data_x: batch_x,
labels: batch_y,
train_flag: False
})
precision[ep] += acc_ / num_iter
sys.stdout.write("\r>> valid %d/%d loss %.4f acc %.4f" %
(step, num_iter, l, acc_))
sys.stdout.write("\n")
# early stop
if ep > 5 and not retrain:
if precision[ep] < 1.2 / self.NUM_CLASSES:
precision = [-1]
break
if 2 * precision[ep] - precision[ep - 5] - precision[
ep - 1] < 0.001 / self.NUM_CLASSES:
precision = precision[:ep]
log += 'early stop at %d epoch\n' % ep
break
cost_time += (float(time.time() - start_time)) / self.epoch
log += 'epoch %d: precision = %.3f, cost time %.3f\n' % (
ep, precision[ep], float(time.time() - start_time))
retrain_log += log
# NAS_LOG << ('eva', retrain_log)
return float(precision)
def test():
train = unpickle('/data/data/cifar-100-python/train')
test = unpickle('/data/data/cifar-100-python/test')
train_data = train[b'data']
test_data = test[b'data']
x_train = train_data.reshape(train_data.shape[0], 3, 32, 32)
x_train = x_train.transpose(0, 2, 3, 1)
y_train = train[b'fine_labels']
x_test = test_data.reshape(test_data.shape[0], 3, 32, 32)
x_test = x_test.transpose(0, 2, 3, 1)
y_test = test[b'fine_labels']
x_train = norm_images(x_train)
x_test = norm_images(x_test)
if not os.path.exists('./trans/tran.tfrecords'):
generate_tfrecord(x_train, y_train, './trans/', 'tran.tfrecords')
generate_tfrecord(x_test, y_test, './trans/', 'test.tfrecords')
dataset = tf.data.TFRecordDataset('./trans/tran.tfrecords')
dataset = dataset.map(parse_function)
dataset = dataset.shuffle(buffer_size=50000)
dataset = dataset.batch(Evaluator().batch_size)
iterator = dataset.make_initializable_iterator()
next_element = iterator.get_next()
x_input = tf.placeholder(tf.float32, [None, 32, 32, 3])
y_input = tf.placeholder(tf.int64, [
None,
])
y_input_one_hot = tf.one_hot(y_input, 100)
lr = tf.placeholder(tf.float32, [])
prob = densenet201(x_input,
reuse=False,
is_training=True,
kernel_initializer=tf.orthogonal_initializer())
loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=prob,
labels=y_input_one_hot))
conv_var = [var for var in tf.trainable_variables() if 'conv' in var.name]
l2_loss = tf.add_n([tf.nn.l2_loss(var) for var in conv_var])
loss = l2_loss * 5e-4 + loss
opt = tf.train.MomentumOptimizer(lr, 0.9, use_nesterov=True)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
train_op = opt.minimize(loss)
logit_softmax = tf.nn.softmax(prob)
acc = tf.reduce_mean(
tf.cast(tf.equal(tf.argmax(logit_softmax, 1), y_input), tf.float32))
config = tf.ConfigProto()
config.allow_soft_placement = True
config.gpu_options.allow_growth = True
now_lr = 0.001 # Warm Up
with tf.Session(config=config) as sess:
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
counter = 0
max_test_acc = -1
for i in range(Evaluator().epoch):
sess.run(iterator.initializer)
while True:
try:
batch_train, label_train = sess.run(next_element)
_, loss_val, acc_val, lr_val = sess.run(
[train_op, loss, acc, lr],
feed_dict={
x_input: batch_train,
y_input: label_train,
lr: now_lr
})
counter += 1
if counter % 100 == 0:
print('counter: ', counter, 'loss_val', loss_val,
'acc: ', acc_val)
except tf.errors.OutOfRangeError:
print('end epoch %d/%d , lr: %f' %
(i, Evaluator().epoch, lr_val))
now_lr = lr_schedule(i)
break
def generate_model(opt):
assert opt.model in [
'resnet', 'preresnet', 'wideresnet', 'resnext', 'densenet'
]
if opt.model == 'resnet':
assert opt.model_depth in [10, 18, 34, 50, 101, 152, 200]
from resnet import get_fine_tuning_parameters
if opt.model_depth == 10:
model = resnet.resnet10(num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration)
elif opt.model_depth == 18:
model = resnet.resnet18(num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration)
elif opt.model_depth == 34:
model = resnet.resnet34(num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration)
elif opt.model_depth == 50:
model = resnet.resnet50(num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration)
elif opt.model_depth == 101:
model = resnet.resnet101(num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration)
elif opt.model_depth == 152:
model = resnet.resnet152(num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration)
elif opt.model_depth == 200:
model = resnet.resnet200(num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration)
elif opt.model == 'wideresnet':
assert opt.model_depth in [50]
from models.wide_resnet import get_fine_tuning_parameters
if opt.model_depth == 50:
model = wide_resnet.resnet50(num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
k=opt.wide_resnet_k,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration)
elif opt.model == 'resnext':
assert opt.model_depth in [50, 101, 152]
from models.resnext import get_fine_tuning_parameters
if opt.model_depth == 50:
model = resnext.resnet50(num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
cardinality=opt.resnext_cardinality,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration)
elif opt.model_depth == 101:
model = resnext.resnet101(num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
cardinality=opt.resnext_cardinality,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration)
elif opt.model_depth == 152:
model = resnext.resnet152(num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
cardinality=opt.resnext_cardinality,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration)
elif opt.model == 'preresnet':
assert opt.model_depth in [18, 34, 50, 101, 152, 200]
from models.pre_act_resnet import get_fine_tuning_parameters
if opt.model_depth == 18:
model = pre_act_resnet.resnet18(
num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration)
elif opt.model_depth == 34:
model = pre_act_resnet.resnet34(
num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration)
elif opt.model_depth == 50:
model = pre_act_resnet.resnet50(
num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration)
elif opt.model_depth == 101:
model = pre_act_resnet.resnet101(
num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration)
elif opt.model_depth == 152:
model = pre_act_resnet.resnet152(
num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration)
elif opt.model_depth == 200:
model = pre_act_resnet.resnet200(
num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration)
elif opt.model == 'densenet':
assert opt.model_depth in [121, 169, 201, 264]
from models.densenet import get_fine_tuning_parameters
if opt.model_depth == 121:
model = densenet.densenet121(num_classes=opt.n_classes,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration)
elif opt.model_depth == 169:
model = densenet.densenet169(num_classes=opt.n_classes,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration)
elif opt.model_depth == 201:
model = densenet.densenet201(num_classes=opt.n_classes,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration)
elif opt.model_depth == 264:
model = densenet.densenet264(num_classes=opt.n_classes,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration)
if not opt.no_cuda:
model = model.cuda()
model = nn.DataParallel(model, device_ids=None)
if opt.pretrain_path:
print('loading pretrained model {}'.format(opt.pretrain_path))
pretrain = torch.load(opt.pretrain_path)
assert opt.arch == pretrain['arch']
model.load_state_dict(pretrain['state_dict'])
if opt.model == 'densenet':
model.module.classifier = nn.Linear(
model.module.classifier.in_features,
opt.n_finetune_classes)
model.module.classifier = model.module.classifier.cuda()
else:
model.module.fc = nn.Linear(model.module.fc.in_features,
opt.n_finetune_classes)
model.module.fc = model.module.fc.cuda()
parameters = get_fine_tuning_parameters(model, opt.ft_begin_index)
return model, parameters
else:
if opt.pretrain_path:
print('loading pretrained model {}'.format(opt.pretrain_path))
pretrain = torch.load(opt.pretrain_path)
assert opt.arch == pretrain['arch']
model.load_state_dict(pretrain['state_dict'])
if opt.model == 'densenet':
model.classifier = nn.Linear(model.classifier.in_features,
opt.n_finetune_classes)
else:
model.fc = nn.Linear(model.fc.in_features,
opt.n_finetune_classes)
parameters = get_fine_tuning_parameters(model, opt.ft_begin_index)
return model, parameters
return model, model.parameters()
def __init__(self,
base_model,
in_channels=3,
out_channels=1,
dropout_rate=0.2,
pretrained=False):
super().__init__()
assert base_model in [
'resnet50', 'resnet101', 'densenet121', 'densenet201',
'efficientnetb0', 'efficientnetb4'
]
self._in_channels = in_channels
self._out_channels = out_channels
if base_model == 'resnet50':
if pretrained:
assert in_channels == 3
self._base_model = resnet50(pretrained=True,
drop_rate=dropout_rate)
else:
self._base_model = resnet50(pretrained=False,
in_channels=in_channels,
drop_rate=dropout_rate)
fc_in_features = 2048
if base_model == 'resnet101':
if pretrained:
assert in_channels == 3
self._base_model = resnet101(pretrained=True,
drop_rate=dropout_rate)
else:
self._base_model = resnet101(pretrained=False,
in_channels=in_channels,
drop_rate=dropout_rate)
fc_in_features = 2048
if base_model == 'densenet121':
if pretrained:
assert in_channels == 3
self._base_model = densenet121(pretrained=True,
drop_rate=dropout_rate)
else:
self._base_model = densenet121(pretrained=False,
drop_rate=dropout_rate,
in_channels=in_channels)
fc_in_features = 1024
if base_model == 'densenet201':
if pretrained:
assert in_channels == 3
self._base_model = densenet201(pretrained=True,
drop_rate=dropout_rate)
else:
self._base_model = densenet201(pretrained=False,
drop_rate=dropout_rate,
in_channels=in_channels)
fc_in_features = 1920
if base_model == 'efficientnetb0':
if pretrained:
assert in_channels == 3
self._base_model = EfficientNet.from_pretrained(
'efficientnet-b0')
else:
self._base_model = EfficientNet.from_name(
'efficientnet-b0', {'in_channels': in_channels})
fc_in_features = 1280
if base_model == 'efficientnetb4':
if pretrained:
assert in_channels == 3
self._base_model = EfficientNet.from_pretrained(
'efficientnet-b4')
else:
self._base_model = EfficientNet.from_name(
'efficientnet-b4', {'in_channels': in_channels})
fc_in_features = 1792
self._fc_mu1 = torch.nn.Linear(fc_in_features, fc_in_features)
self._fc_mu2 = torch.nn.Linear(fc_in_features, out_channels)
self._fc_logvar1 = torch.nn.Linear(fc_in_features, fc_in_features)
# self._fc_logvar2 = torch.nn.Linear(fc_in_features, out_channels)
self._fc_logvar2 = torch.nn.Linear(fc_in_features, 1)
if 'resnet' in base_model:
self._base_model.fc = torch.nn.Identity()
elif 'densenet' in base_model: # densenet
self._base_model.classifier = torch.nn.Identity()
elif 'efficientnet' in base_model:
self._base_model._fc = torch.nn.Identity()
self._dropout_T = 25
self._dropout_p = 0.5
def findBadAppleinDualDenseExtClassifier(extClassiferPath, loader):
# modelPath = r'C:\Users\wzuo\Developer\ML for APT\models\1534191799.996107.model'
# patchModelPath=r'C:\Users\wzuo\Developer\ML for APT\models\1534191799.996107.patchModel'
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
wrongs = []
#model = MD.densenet201()
model_global = MD.densenet201(pretrained=True)
model_local = MD.densenet201(pretrained=True)
# todo change model init
#patchModel =MD.SimpleNet()
# todo change second model init
#num_ftrs = model.classifier.in_features
num_ftrs_global = model_global.classifier.in_features
num_ftrs_local = model_local.classifier.in_features
externalClassifier = nn.Sequential(
nn.Linear(num_ftrs_global + num_ftrs_local + 1, 128), nn.ReLU(),
nn.Linear(128, 2))
# externalClassifier = nn.Sequential(
# nn.Linear(num_ftrs_global + num_ftrs_local + 1, 2)
# )
# todo init the external classifier instead of the whole two densenet model
#the_model = TheModelClass(*args, **kwargs)
#model.load_state_dict(torch.load(modelPath))
externalClassifier.load_state_dict(torch.load(extClassiferPath))
# todo load the params in the external classifier
#patchModel.load_state_dict(torch.load(patchModelPath))
#model = model.to(device)
#patchModel = patchModel.to(device)
model_global = model_global.to(device)
model_local = model_local.to(device)
externalClassifier = externalClassifier.to(device)
# todo send the external classifer to the device
#patchModel.eval()
#model.eval()
model_local.eval()
model_global.eval()
externalClassifier.eval()
# set all separate model to eval()
for batch_idx, sample in enumerate(loader):
images = sample['image'].to(device)
patchs = sample['patch'].to(device)
ages = sample['age'].to(device)
labels = sample['label'].to(device)
ids = sample['id']
outputLocal = model_local(patchs)
outputGlobal = model_global(images)
interim = torch.cat((outputGlobal, outputLocal), 1)
interim = torch.cat((ages.unsqueeze(1), interim), 1)
outputs = externalClassifier(interim)
_, preds = torch.max(outputs, 1)
#wrongs.append(ids[preds != labels.data])
#print(ids)
#print(preds != labels.data)
if (preds != labels.data)[0]:
wrongs.append(ids[0])
# save the entire sample
return wrongs
def main():
net = densenet.densenet201(sample_size=64,
sample_duration=30,
num_classes=num_classes)
if pretrained:
pretrained_weights = torch.load(pretrained_path)
net.load_state_dict(pretrained_weights['state_dict'])
train_dataset = MatrixDataset(base_dir='../data/', num_classes=num_classes)
# train_loader = DataLoader(train_dataset, batch_size=batch_size, num_workers=num_workers)
# test_dataset = MatrixDataset(base_dir='../data/test/', mode='test', sample_size=sample_size, num_classes=num_classes)
# test_loader = DataLoader(test_dataset, batch_size=batch_size, num_workers=num_workers)
print len(train_dataset), "Train + Val size"
# print len(test_dataset)
train_loader, test_loader, class_weights = get_train_valid_loader(
train_dataset, train_size=0.7,
shuffle=False) #shuffle=False since sampler takes continous inputs
if sample_size:
class_weights = np.ones((num_classes), dtype=np.float32)
criterion = nn.CrossEntropyLoss()
else:
# class_weights = get_class_weights(train_dataset, train_loader)
print class_weights
criterion = nn.CrossEntropyLoss(class_weights)
optimizer = optim.SGD(filter(lambda p: p.requires_grad, net.parameters()),
lr=lr,
momentum=momentum,
weight_decay=weight_decay)
make_dir(weights_folder)
make_dir(plot_folder)
if use_cuda:
net.cuda()
### Plot Lists
if resume == False:
# for replacing whole plots
train_loss = []
train_loss_epoch = []
precision_train = []
recall_train = []
avg_precision_train = []
avg_recall_train = []
f1_scores_train = []
precision_test = []
recall_test = []
avg_precision_test = []
avg_recall_test = []
f1_scores_test = []
test_loss = []
test_loss_mean = []
start_epoch = 0
best_prec1 = 0
early_stop_counter = 0
### Resuming checkpoints
if resume:
checkpoint = torch.load(checkpoint_path)
net.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
start_epoch = checkpoint['epoch'] # gives next epoch number
best_prec1 = checkpoint['best_prec1']
for epoch in xrange(start_epoch, epochs):
### As I need shuffled dataset in every epoch!
# train_dataset = AnatomyDataset(base_dir='../data/train/', mode='train', sample_size=sample_size, num_classes=num_classes)
# train_loader = DataLoader(train_dataset, batch_size=batch_size, num_workers=num_workers)
### Plot lists
if resume:
# for appending to the plots
train_loss = []
train_loss_epoch = []
precision_train = []
recall_train = []
avg_precision_train = []
avg_recall_train = []
f1_scores_train = []
precision_test = []
recall_test = []
avg_precision_test = []
avg_recall_test = []
f1_scores_test = []
test_loss = []
test_loss_mean = []
print "------------------------TRAINING {} - EPOCH: {}---------------------------".format(
modelname, epoch)
net.train()
output_labels = []
target_labels = []
temp_loss_epoch = []
train_loss, temp_loss_epoch, output_labels, target_labels = train(
net, criterion, optimizer, epoch, train_loader, temp_loss_epoch,
train_loss, output_labels, target_labels)
### Accuracies
train_loss_epoch.append(sum(temp_loss_epoch) / len(temp_loss_epoch))
precision_train.append(
np.asarray(
precision_score(target_labels,
output_labels,
labels,
average=None)))
recall_train.append(
np.asarray(
recall_score(target_labels,
output_labels,
labels,
average=None)))
avg_precision_train.append(
precision_score(target_labels,
output_labels,
labels,
average='weighted'))
avg_recall_train.append(
recall_score(target_labels,
output_labels,
labels,
average='weighted'))
f1_scores_train.append(
f1_score(target_labels, output_labels, labels, average='weighted'))
### Printing
print "precision_train ", precision_train[-1]
print "recall_train ", recall_train[-1]
print "train_loss_epoch", train_loss_epoch[-1]
print "avg_precision_train ", avg_precision_train[-1]
print "avg_recall_train ", avg_precision_train[-1]
print "f1_scores_train ", f1_scores_train[-1]
### Plotting
plot_visdom(epoch, train_loss, win='train_loss')
plot_visdom(epoch, train_loss_epoch, win='train_loss_epoch')
plot_visdom(epoch, precision_train, win='precision_train')
plot_visdom(epoch, recall_train, win='recall_train')
plot_visdom(epoch, avg_precision_train, win='avg_precision_train')
plot_visdom(epoch, avg_recall_train, win='avg_recall_train')
plot_visdom(epoch, f1_scores_train, win='f1_scores_train')
##### Testing ######
print "------------------------TESTING EPOCH: {}---------------------------".format(
epoch)
net.eval()
output_labels = []
target_labels = []
temp_loss_epoch = []
test_loss, temp_loss_epoch, output_labels, target_labels = test(
net, criterion, optimizer, epoch, test_loader, temp_loss_epoch,
test_loss, output_labels, target_labels)
### Accuracies
test_loss_mean.append(sum(temp_loss_epoch) / len(temp_loss_epoch))
precision_test.append(
np.asarray(
precision_score(target_labels,
output_labels,
labels,
average=None)))
recall_test.append(
np.asarray(
recall_score(target_labels,
output_labels,
labels,
average=None)))
avg_precision_test.append(
precision_score(target_labels,
output_labels,
labels,
average='weighted'))
avg_recall_test.append(
recall_score(target_labels,
output_labels,
labels,
average='weighted'))
f1_scores_test.append(
f1_score(target_labels, output_labels, labels, average='weighted'))
### Printing logs
print "test loss mean ", test_loss_mean[-1]
print "precision_test ", precision_test[-1]
print "recall_test ", recall_test[-1]
print "avg_precision_test ", avg_precision_test[-1]
print "avg_recall_test ", avg_precision_test[-1]
print "f1_scores_test ", f1_scores_test[-1]
### Plotting
epochm = epoch
plot_visdom(epochm, test_loss, win='test_loss')
plot_visdom(epochm, test_loss_mean, win='test_loss_mean')
plot_visdom(epochm, precision_test, win='precision_test')
plot_visdom(epochm, recall_test, win='recall_test')
plot_visdom(epochm, avg_precision_test, win='avg_precision_test')
plot_visdom(epochm, avg_recall_test, win='avg_recall_test')
plot_visdom(epochm, f1_scores_test, win='f1_scores_test')
### Save the environment
vis.save(envs=[basename + modelname])
### Checking if best
# prec1 = test_loss_mean[-1]
prec1 = f1_scores_test[-1]
is_best = prec1 > best_prec1
best_prec1 = max(prec1, best_prec1)
### Saving weights
if is_best or epoch % 10 == 0 or epoch == epochs - 1:
save_checkpoint(
{
'epoch': epoch + 1,
'arch': modelname,
'state_dict': net.state_dict(),
'best_prec1': best_prec1,
'optimizer': optimizer.state_dict(),
}, is_best)
def get_model(args):
network = args.network
if network == 'vgg11':
model = vgg.vgg11(num_classes=args.class_num)
elif network == 'vgg13':
model = vgg.vgg13(num_classes=args.class_num)
elif network == 'vgg16':
model = vgg.vgg16(num_classes=args.class_num)
elif network == 'vgg19':
model = vgg.vgg19(num_classes=args.class_num)
elif network == 'vgg11_bn':
model = vgg.vgg11_bn(num_classes=args.class_num)
elif network == 'vgg13_bn':
model = vgg.vgg13_bn(num_classes=args.class_num)
elif network == 'vgg16_bn':
model = vgg.vgg16_bn(num_classes=args.class_num)
elif network == 'vgg19_bn':
model = vgg.vgg19_bn(num_classes=args.class_num)
elif network == 'resnet18':
model = models.resnet18(num_classes=args.class_num)
model.conv1 = torch.nn.Conv2d(in_channels=1,
out_channels=model.conv1.out_channels,
kernel_size=model.conv1.kernel_size,
stride=model.conv1.stride,
padding=model.conv1.padding,
bias=model.conv1.bias)
elif network == 'resnet34':
model = models.resnet34(num_classes=args.class_num)
model.conv1 = torch.nn.Conv2d(in_channels=1,
out_channels=model.conv1.out_channels,
kernel_size=model.conv1.kernel_size,
stride=model.conv1.stride,
padding=model.conv1.padding,
bias=model.conv1.bias)
elif network == 'resnet50':
model = models.resnet50(num_classes=args.class_num)
model.conv1 = torch.nn.Conv2d(in_channels=1,
out_channels=model.conv1.out_channels,
kernel_size=model.conv1.kernel_size,
stride=model.conv1.stride,
padding=model.conv1.padding,
bias=model.conv1.bias)
elif network == 'resnet101':
model = models.resnet101(num_classes=args.class_num)
model.conv1 = torch.nn.Conv2d(in_channels=1,
out_channels=model.conv1.out_channels,
kernel_size=model.conv1.kernel_size,
stride=model.conv1.stride,
padding=model.conv1.padding,
bias=model.conv1.bias)
elif network == 'resnet152':
model = models.resnet152(num_classes=args.class_num)
model.conv1 = torch.nn.Conv2d(in_channels=1,
out_channels=model.conv1.out_channels,
kernel_size=model.conv1.kernel_size,
stride=model.conv1.stride,
padding=model.conv1.padding,
bias=model.conv1.bias)
elif network == 'densenet121':
model = densenet.densenet121(num_classes=args.class_num)
elif network == 'densenet169':
model = densenet.densenet169(num_classes=args.class_num)
elif network == 'densenet161':
model = densenet.densenet161(num_classes=args.class_num)
elif network == 'densenet201':
model = densenet.densenet201(num_classes=args.class_num)
return model
import densenet MatrixNet = densenet.densenet201()
def generate_model(opt):
assert opt.mode in ['score', 'feature']
if opt.mode == 'score':
last_fc = True
elif opt.mode == 'feature':
last_fc = False
assert opt.model_name in [
'resnet', 'preresnet', 'wideresnet', 'resnext', 'densenet'
]
if opt.model_name == 'resnet':
assert opt.model_depth in [10, 18, 34, 50, 101, 152, 200]
if opt.model_depth == 10:
model = resnet.resnet10(num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration,
last_fc=last_fc)
elif opt.model_depth == 18:
model = resnet.resnet18(num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration,
last_fc=last_fc)
elif opt.model_depth == 34:
model = resnet.resnet34(num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration,
last_fc=last_fc)
elif opt.model_depth == 50:
model = resnet.resnet50(num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration,
last_fc=last_fc)
elif opt.model_depth == 101:
model = resnet.resnet101(num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration,
last_fc=last_fc)
elif opt.model_depth == 152:
model = resnet.resnet152(num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration,
last_fc=last_fc)
elif opt.model_depth == 200:
model = resnet.resnet200(num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration,
last_fc=last_fc)
elif opt.model_name == 'wideresnet':
assert opt.model_depth in [50]
if opt.model_depth == 50:
model = wide_resnet.resnet50(num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
k=opt.wide_resnet_k,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration,
last_fc=last_fc)
elif opt.model_name == 'resnext':
assert opt.model_depth in [50, 101, 152]
if opt.model_depth == 50:
model = resnext.resnet50(num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
cardinality=opt.resnext_cardinality,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration,
last_fc=last_fc)
elif opt.model_depth == 101:
model = resnext.resnet101(num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
cardinality=opt.resnext_cardinality,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration,
last_fc=last_fc)
elif opt.model_depth == 152:
model = resnext.resnet152(num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
cardinality=opt.resnext_cardinality,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration,
last_fc=last_fc)
elif opt.model_name == 'preresnet':
assert opt.model_depth in [18, 34, 50, 101, 152, 200]
if opt.model_depth == 18:
model = pre_act_resnet.resnet18(
num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration,
last_fc=last_fc)
elif opt.model_depth == 34:
model = pre_act_resnet.resnet34(
num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration,
last_fc=last_fc)
elif opt.model_depth == 50:
model = pre_act_resnet.resnet50(
num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration,
last_fc=last_fc)
elif opt.model_depth == 101:
model = pre_act_resnet.resnet101(
num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration,
last_fc=last_fc)
elif opt.model_depth == 152:
model = pre_act_resnet.resnet152(
num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration,
last_fc=last_fc)
elif opt.model_depth == 200:
model = pre_act_resnet.resnet200(
num_classes=opt.n_classes,
shortcut_type=opt.resnet_shortcut,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration,
last_fc=last_fc)
elif opt.model_name == 'densenet':
assert opt.model_depth in [121, 169, 201, 264]
if opt.model_depth == 121:
model = densenet.densenet121(num_classes=opt.n_classes,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration,
last_fc=last_fc)
elif opt.model_depth == 169:
model = densenet.densenet169(num_classes=opt.n_classes,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration,
last_fc=last_fc)
elif opt.model_depth == 201:
model = densenet.densenet201(num_classes=opt.n_classes,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration,
last_fc=last_fc)
elif opt.model_depth == 264:
model = densenet.densenet264(num_classes=opt.n_classes,
sample_size=opt.sample_size,
sample_duration=opt.sample_duration,
last_fc=last_fc)
if not opt.no_cuda:
model = model.cuda()
model = nn.DataParallel(model, device_ids=None)
return model
from dataset import *
### Dataset configs ###
num_classes = 24
use_cuda = torch.cuda.is_available()
### trained model configs ###
modelname = "3d_model"
trained_path = 'Weights/mc-cls/model_best.pth.tar'
batch_size = 256
if __name__ == '__main__':
net = densenet.densenet201(sample_size=64, sample_duration=30, num_classes=num_classes)
trained_weights = torch.load(trained_path)
net.load_state_dict(trained_weights['state_dict'])
test_dataset = MatrixDataset(base_dir='../data/', num_classes=num_classes, mode='test')
test_loader = DataLoader(test_dataset, batch_size=batch_size)
output_scores = []
if use_cuda:
net.cuda()
net.eval()
for batch,(vid, vidname) in enumerate(test_loader):
print "Processing batch {}/{}".format(batch,len(test_loader))
#patchModel.load_state_dict(best_patchModel_wts)
extClassifier.load_state_dict(best_classifier_wts)
model_serial = str(datetime.now().timestamp())
# torch.save(model.state_dict(),
# os.path.join(r'C:\Users\wzuo\Developer\ML for APT\models', model_serial + '.model'))
# torch.save(patchModel.state_dict(),
# os.path.join(r'C:\Users\wzuo\Developer\ML for APT\models', model_serial + '.patchModel'))
torch.save(extClassifier.state_dict(),
os.path.join(model_path, model_serial + '.clsmodel'))
with open(os.path.join(model_path, model_serial + '.json'), 'w') as fp:
json.dump(param_dict, fp)
return extClassifier
model_global = MD.densenet201(pretrained=True)
model_local = MD.densenet169(pretrained=True)
param_dict['model_global'] = 'densenet201'
param_dict['model_local'] = 'densenet169'
# todo change architecture from here
#patch_model = MD.densenet121(pretrained=True)
#patch_model = MD.SimpleNet()
#param_dict['patch_base'] = 'densenet121'
for param in model_global.parameters():
param.requires_grad = False
for param in model_local.parameters():
param.requires_grad = False
def get_model(class_weights=None):
model = densenet201(pretrained=True, drop_rate=0.25, final_drop_rate=0.25)
# model_size: penultimate_layer_output_dim,
# 201: 1920, 169: 1664, 121: 1024
# make all params untrainable
for p in model.parameters():
p.requires_grad = False
# reset the last fc layer
model.classifier = nn.Linear(1920, 256)
normal(model.classifier.weight, 0.0, 0.001)
constant(model.classifier.bias, 0.0)
# make some other params trainable
trainable_params = []
trainable_params += [
n for n, p in model.named_parameters() if 'norm5' in n
]
trainable_params += [
n for n, p in model.named_parameters() if 'denseblock4' in n
]
for n, p in model.named_parameters():
if n in trainable_params:
p.requires_grad = True
for m in model.features.denseblock4.modules():
if isinstance(m, nn.ReLU):
m.inplace = False
# create different parameter groups
classifier_weights = [model.classifier.weight]
classifier_biases = [model.classifier.bias]
features_weights = [
p for n, p in model.named_parameters()
if n in trainable_params and 'conv' in n
]
features_bn_weights = [
p for n, p in model.named_parameters()
if n in trainable_params and 'norm' in n and 'weight' in n
]
features_bn_biases = [
p for n, p in model.named_parameters()
if n in trainable_params and 'bias' in n
]
# you can set different learning rates
classifier_lr = 1e-2
features_lr = 1e-2
# but they are not actually used (because lr_scheduler is used)
params = [{
'params': classifier_weights,
'lr': classifier_lr,
'weight_decay': 1e-4
}, {
'params': classifier_biases,
'lr': classifier_lr
}, {
'params': features_weights,
'lr': features_lr,
'weight_decay': 1e-4
}, {
'params': features_bn_weights,
'lr': features_lr
}, {
'params': features_bn_biases,
'lr': features_lr
}]
optimizer = optim.SGD(params, momentum=0.9, nesterov=True)
# loss function
criterion = nn.CrossEntropyLoss(weight=class_weights).cuda()
# move the model to gpu
model = model.cuda()
return model, criterion, optimizer