def train(file_train, file_test, epochs, batch_size, input_dim_size,
hidden_dim_size, output_dim_size, learning_rate):
model = NeuralNetwork(input_dim_size, hidden_dim_size, output_dim_size)
optimizer = utils.SGD(model, learning_rate)
train_loader = load.DataLoader(file_train)
train_features, train_labels = train_loader.load()
for epoch in range(epochs):
for minibatch_features, minibatch_labels in utils.get_batches(
train_features, train_labels, batch_size, shuffle=True):
# 順伝播
minibatch_features_reshaped = minibatch_features.T
z1, minibatch_predicted_labels = model.forward(
minibatch_features_reshaped)
# 逆伝播
grads = model.backward(x=minibatch_features,
z1=z1,
y=minibatch_predicted_labels,
t=minibatch_labels)
# パラメータの更新
optimizer.update(grads)
# テストデータによる Inference と評価
accuracy = inference_test.infer(file_test=file_test,
model_trained=model)
print('[{}] EPOCH {} Accuracy:{:.8f}'.format(datetime.datetime.today(),
epoch, accuracy))
print('[{}] Finished Training'.format(datetime.datetime.today()))
return model
def train():
batch_size = 256
train_data, test_data = utils.load_data_fashion_mnist(batch_size)
softmax_cross_entropy = gluon.loss.SoftmaxCrossEntropyLoss()
learning_rate = 0.2
for epoch in range(5):
train_loss = 0.
train_acc = 0.
for data, label in train_data:
label = label.as_in_context(ctx)
with autograd.record():
output = net(data, is_training=True)
loss = softmax_cross_entropy(output, label)
loss.backward()
utils.SGD(params, learning_rate / batch_size)
train_loss += nd.mean(loss).asscalar()
train_acc += utils.accuracy(output, label)
test_acc = utils.evaluate_accuracy(test_data, net, ctx)
print("Epoch %d. Loss: %f, Train acc %f, Test acc %f" %
(epoch, train_loss / len(train_data),
train_acc / len(train_data), test_acc))
def train_and_predict_rnn(rnn, is_random_iter, epochs, num_steps, hidden_dim,
learning_rate, clipping_theta, batch_size,
pred_period, pred_len, seqs, get_params, get_inputs,
ctx, corpus_indices, idx_to_char, char_to_idx,
is_lstm=False):
if is_random_iter:
data_iter = data_iter_random
else:
data_iter = data_iter_consecutive
params = get_params()
softmax_cross_entropy = gluon.loss.SoftmaxCrossEntropyLoss()
for e in range(1, epochs+1):
# 如用相邻批量采样，在同一个 epoch 中，隐含变量只需要在该 epoch 开始的时候初始化。
if not is_random_iter:
state_h = nd.zeros(shape=(batch_size, hidden_dim), ctx=ctx)
if is_lstm:
# 当 RNN 使用LSTM时才会用到，这里可以忽略
state_c = nd.zeros(shape=(batch_size, hidden_dim), ctx=ctx)
train_loss, num_examples = 0, 0
for data, label in data_iter(corpus_indices, batch_size, num_steps, ctx):
# 如使用随机批量采样，处理每个随机小批量前都需要初始化隐含变量。
if is_random_iter:
state_h = nd.zeros(shape=(batch_size, hidden_dim), ctx=ctx)
if is_lstm:
# 当 RNN 使用 LSTM 时才会用到，这里可以忽略。
state_c = nd.zeros(shape=(batch_size, hidden_dim), ctx=ctx)
with autograd.record():
# outputs 尺寸： (batch_size, vocab_size)
if is_lstm:
# 当RNN 使用 LSTM 时才会用到，这里可以忽略。
outputs, state_h, state_c = rnn(
get_inputs(data), state_h, state_c, *params)
else:
outputs, state_h = rnn(get_inputs(data), state_h, *params)
# 设 t_ib_j 为 i 时间批量中的 j 元素：
# label 尺寸： (batch_size * num_steps)
# label = [t_0b_0, t_0b_1, ... , t_1b_0, t_1b_1, ..., ]
label = label.T.reshape((-1,))
# 拼接 outputs, 尺寸: (batch_size * num_steps, vocab_size)
outputs = nd.concat(*outputs, dim=0)
# 经上述操作，outputs 和 label 已对齐
loss = softmax_cross_entropy(outputs, label)
loss.backward()
grad_clipping(params, clipping_theta, ctx)
utils.SGD(params, learning_rate)
train_loss += nd.sum(loss).asscalar()
num_examples += loss.size
if e % pred_period == 0:
print("Epoch %d. Perplexity %f" %
(e, exp(train_loss/num_examples)))
for seq in seqs:
print(' - ', predict_rnn(rnn, seq, pred_len, params,
hidden_dim, ctx, idx_to_char, char_to_idx, get_inputs, is_lstm))
print()
def train_batch(data, label, params, ctx, lr):
data_list = split_and_load(data, ctx)
label_list = split_and_load(label, ctx)
with autograd.record():
losses = [loss(lenet(X, W), Y) for X, W, Y in zip(data_list, params, label_list)]
for l in losses:
l.backward()
for i in range(len(params[0])):
allreduce(params[c][i].grad for c in range(len(ctx)))
for p in params:
utils.SGD(p, lr / data.shape[0])
def train():
for epoch in range(5):
train_loss = 0.
train_acc = 0.
for data, label in train_data:
with autograd.record():
output = net(data)
loss = softmax_cross_entropy(output, label)
loss.backward()
utils.SGD(params, learning_rate / batch_size)
train_loss += nd.mean(loss).asscalar()
train_acc += utils.accuracy(output, label)
test_acc = utils.evaluate_accuracy(test_data, net)
print("Epoch %d. Loss: %f, Train acc %f, Test acc %f" %
(epoch, train_loss / len(train_data),
train_acc / len(train_data), test_acc))
h2 = dropout(h2,drop_prob2)
return nd.dot(h2,W3)+b3
from mxnet import autograd
from mxnet import gluon
softmax_cross_entropy = gluon.loss.SoftmaxCrossEntropyLoss()
learning_rate = .5
epochs = 10
for epoch in range(epochs):
train_loss = 0.
train_acc = 0.
for data,label in train_data:
with autograd.record():
output = net(data)
loss = softmax_cross_entropy(output,label)
loss.backward()
utils.SGD(params,learning_rate/batch_size)
train_loss += nd.mean(loss).asscalar()
train_acc +=utils.accuracy(output,label)
test_acc = utils.evaluate_accuracy(test_data,net)
print("Epoch: %d. train loss: %f, train acc: %f, test acc: %f"%(epoch,train_loss/len(train_data),train_acc/len(train_data),test_acc))
def train_and_predict_rnn(rnn, is_random_iter, epochs, num_steps, hidden_dim,
learning_rate, clipping_norm, batch_size,
pred_period, pred_len, seqs, get_params, get_inputs,
ctx, corpus_indices, idx_to_char, char_to_idx,
is_lstm=False):
"""Train an RNN model and predict the next item in the sequence."""
if is_random_iter:
data_iter = data_iter_random
else:
data_iter = data_iter_consecutive
params = get_params()
softmax_cross_entropy = gluon.loss.SoftmaxCrossEntropyLoss()
for e in range(1, epochs + 1):
# If consecutive sampling is used, in the same epoch, the hidden state
# is initialized only at the beginning of the epoch.
if not is_random_iter:
state_h = nd.zeros(shape=(batch_size, hidden_dim), ctx=ctx)
if is_lstm:
state_c = nd.zeros(shape=(batch_size, hidden_dim), ctx=ctx)
train_loss, num_examples = 0, 0
for data, label in data_iter(corpus_indices, batch_size, num_steps,
ctx):
# If random sampling is used, the hidden state has to be
# initialized for each mini-batch.
if is_random_iter:
state_h = nd.zeros(shape=(batch_size, hidden_dim), ctx=ctx)
if is_lstm:
state_c = nd.zeros(shape=(batch_size, hidden_dim), ctx=ctx)
with autograd.record():
# outputs shape: (batch_size, vocab_size)
if is_lstm:
outputs, state_h, state_c = rnn(get_inputs(data), state_h,
state_c, *params)
else:
outputs, state_h = rnn(get_inputs(data), state_h, *params)
# Let t_ib_j be the j-th element of the mini-batch at time i.
# label shape: (batch_size * num_steps)
# label = [t_0b_0, t_0b_1, ..., t_1b_0, t_1b_1, ..., ].
label = label.T.reshape((-1,))
# Concatenate outputs:
# shape: (batch_size * num_steps, vocab_size).
outputs = nd.concat(*outputs, dim=0)
# Now outputs and label are aligned.
loss = softmax_cross_entropy(outputs, label)
loss.backward()
grad_clipping(params, clipping_norm, ctx)
utils.SGD(params, learning_rate)
train_loss += nd.sum(loss).asscalar()
num_examples += loss.size
if e % pred_period == 0:
print("Epoch %d. Training perplexity %f" % (e,
exp(train_loss/num_examples)))
for seq in seqs:
print(' - ', predict_rnn(rnn, seq, pred_len, params,
hidden_dim, ctx, idx_to_char, char_to_idx, get_inputs,
is_lstm))
print()
def predict(args):
nb_classes = 1
orig_width = 1918
orig_height = 1280
model_name = str(args.model)
model_weight = str(args.model_weight)
output_path = str(args.out_path)
input_path = str(args.in_path)
batch_size = int(args.batchsize)
image_size = int(args.image_size)
df_test = pd.read_csv(input_path + '/sample_submission.csv')
ids_test = df_test['img'].map(lambda s: s.split('.')[0])
image_size = 256
threshold = 0.5
if model_name == 'unet128':
image_size = 128
batch_size = 48
model = get_unet_128()
model.load_weights(filepath=model_weight)
# tested
# elif model_name == 'unet256':
# image_size = 256
# batch_size = 24
# model = get_unet_256()
# model.load_weights(filepath=model_weight)
# elif model_name == 'unet512':
# image_size = 512
# batch_size = 12
# model = get_unet_512()
# model.load_weights(filepath=model_weight)
elif model_name == 'unet1024':
img_tuple = (256, 256, 3)
batch_size = 64
model = get_unet_1024(input_shape=img_tuple, num_classes=nb_classes)
model.compile(optimizer=utils.SGD(lr=1e-02,
decay=1e-6,
momentum=0.9,
nesterov=True,
accumulator=np.ceil(
float(20) / float(batch_size))),
loss=weighted_loss,
metrics=['binary_accuracy', dice_loss])
model.load_weights(filepath=model_weight)
# elif model_name == 'segnet':
# image_size = 256
# batch_size = 28
# model = SegNet(input_shape=(image_size, image_size, 3), classes=nb_classes)
# model.compile(optimizer=optimizers.Adam(lr=0.01), loss=bce_dice_loss, metrics=['binary_accuracy',
# dice_loss])
#
# model.load_weights(filepath=model_weight)
# elif model_name == 'enet_unpooling':
# model_path = output_path + 'Enet_unpooling_{val_loss:.4f}-{val_binary_accuracy:.4f}.hdf5'
# csv_path = output_path + 'Enet_unpooling.csv'
# image_size = int(256)
# model = get_enet_unpooling(input_shape=(image_size, image_size, 3), classes=nb_classes)
# model.compile(optimizer=optimizers.Adam(lr=0.01), loss=losses.binary_crossentropy, metrics=['binary_accuracy',
# dice_loss])
elif model_name == 'enet_naive_upsampling':
img_tuple = (512, 512, 3)
batch_size = 4
model = get_enet_naive_upsampling(input_shape=img_tuple,
classes=nb_classes)
model.compile(optimizer=utils.SGD(lr=1e-02,
decay=1e-6,
momentum=0.9,
nesterov=True,
accumulator=5),
loss=weighted_loss,
metrics=['binary_accuracy', dice_loss])
model.load_weights(filepath=model_weight)
# elif model_name == 'init_unet':
# img_tuple = (1024, 1024, 3)
# batch_size = 6
# model = get_init_unet(input_shape=img_tuple, num_classes=nb_classes)
# model.compile(optimizer=utils.SGD(lr=1e-02, decay=1e-6, momentum=0.9, nesterov=True, accumulator=5),
# loss=weighted_loss,
# metrics=['binary_accuracy', dice_loss])
# model.load_weights(filepath=model_weight)
#
# elif model_name == 'unet':
# img_tuple = (256, 256, 3)
# batch_size = 24
# model = get_unet(input_shape=img_tuple, num_classes=nb_classes)
# model.compile(optimizer=utils.SGD(lr=1e-02, decay=1e-6, momentum=0.9, nesterov=True, accumulator=5),
# loss=weighted_loss,
# metrics=['binary_accuracy', dice_loss])
# model.load_weights(filepath=model_weight)
elif model_name == 'modify_unet':
img_tuple = (512, 512, 3)
batch_size = 5
model = get_modify_unet(input_shape=img_tuple, num_classes=nb_classes)
model.compile(optimizer=utils.SGD(lr=1e-02,
decay=1e-6,
momentum=0.9,
nesterov=True,
accumulator=5),
loss=weighted_loss,
metrics=['binary_accuracy', dice_loss])
model.load_weights(filepath=model_weight)
# elif model_name == 'modify_unet_V2':
# img_tuple = (1024, 1024, 3)
# batch_size = 4
# model = get_modify_unet_V2(input_shape=img_tuple, num_classes=nb_classes)
# model.compile(optimizer=utils.SGD(lr=1e-02, decay=1e-6, momentum=0.9, nesterov=True, accumulator=5),
# loss=weighted_loss,
# metrics=['binary_accuracy', dice_loss])
# model.load_weights(filepath=model_weight)
#
# elif model_name == 'linknet':
# img_tuple = (1024, 1024, 3)
# batch_size = 6
#
# base_model = LinkNet(input_shape=img_tuple, classes=nb_classes)
# classify = Conv2D(nb_classes, (1, 1), activation='sigmoid')(base_model.output)
# model = Model(inputs=base_model.input, outputs=classify)
# model.compile(optimizer=utils.SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True, accumulator=10),
# loss=weighted_loss,
# metrics=['binary_accuracy', dice_loss])
#
# model.load_weights(filepath=model_weight)
elif model_name == 'linknet_res34':
img_tuple = (256, 256, 3)
batch_size = 84
base_model = LinkNet_Res34(input_shape=img_tuple, classes=nb_classes)
classify = Conv2D(nb_classes, (1, 1),
activation='sigmoid')(base_model.output)
model = Model(inputs=base_model.input, outputs=classify)
model.compile(optimizer=utils.SGD(lr=0.01,
decay=1e-6,
momentum=0.9,
nesterov=True,
accumulator=10),
loss=weighted_loss,
metrics=['binary_accuracy', dice_loss])
model.load_weights(filepath=model_weight)
# elif model_name == 'linknet_res50':
# img_tuple = (1280, 1280, 3)
# batch_size = 4
#
# base_model = LinkNet_Res50(input_shape=img_tuple, classes=nb_classes)
# classify = Conv2D(nb_classes, (1, 1), activation='sigmoid')(base_model.output)
# model = Model(inputs=base_model.input, outputs=classify)
# model.compile(optimizer=utils.SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True, accumulator=10),
# loss=weighted_loss,
# metrics=['binary_accuracy', dice_loss])
#
# model.load_weights(filepath=model_weight)
names = []
for id in ids_test:
names.append('{}.jpg'.format(id))
rles = []
print('Predicting on {} samples with batch_size = {}...'.format(
len(ids_test), batch_size))
print('Conditions low quality %r' % (args.lowq, ))
for start in tqdm(range(0, len(ids_test), batch_size)):
x_batch = []
end = min(start + batch_size, len(ids_test))
ids_test_batch = ids_test[start:end]
for id in ids_test_batch.values:
if args.lowq:
img = cv2.imread(input_path + 'test/{}.jpg'.format(id))
else:
img = cv2.imread(input_path + 'test_hq/{}.jpg'.format(id))
img = cv2.resize(img, (img_tuple[1], img_tuple[0]))
x_batch.append(img)
x_batch = np.array(x_batch, np.float32) / 255
preds = model.predict_on_batch(x_batch)
preds = np.squeeze(preds, axis=3)
for pred in preds:
prob = cv2.resize(pred, (orig_width, orig_height))
mask = prob > threshold
rle = run_length_encode(mask)
rles.append(rle)
print("Generating submission file...")
df = pd.DataFrame({'img': names, 'rle_mask': rles})
df.to_csv(output_path + model_weight.split('/')[-1] + '.csv.gz',
index=False,
compression='gzip')
def train_and_predict_rnn(rnn,
is_random_iter,
epochs,
num_steps,
hidden_dim,
learning_rate,
clipping_theta,
batch_size,
pred_period,
pred_len,
seqs,
get_params,
get_inputs,
ctx,
corpus_indices,
idx_to_char,
char_to_idx,
is_lstm=False):
if is_random_iter:
data_iter = data_iter_random
else:
data_iter = data_iter_consecutive
params = get_params()
softmax_cross_entropy = gluon.loss.SoftmaxCrossEntropyLoss()
ans_collect = []
for e in range(1, epochs + 1):
# 如使用相邻批量采样，在同一个epoch中，隐含变量只需要在该epoch开始的时候初始化。
if not is_random_iter:
state_h = nd.zeros(shape=(batch_size, hidden_dim), ctx=ctx)
if is_lstm:
# 当RNN使用LSTM时才会用到，这里可以忽略。
state_c = nd.zeros(shape=(batch_size, hidden_dim), ctx=ctx)
train_loss, num_examples = 0, 0
for data, label in data_iter(corpus_indices, batch_size, num_steps,
ctx):
# 如使用随机批量采样，处理每个随机小批量前都需要初始化隐含变量。
if is_random_iter:
state_h = nd.zeros(shape=(batch_size, hidden_dim), ctx=ctx)
if is_lstm:
# 当RNN使用LSTM时才会用到，这里可以忽略。
state_c = nd.zeros(shape=(batch_size, hidden_dim), ctx=ctx)
with autograd.record():
# outputs 尺寸：(batch_size, vocab_size)
if is_lstm:
# 当RNN使用LSTM时才会用到，这里可以忽略。
outputs, state_h, state_c = rnn(get_inputs(data), state_h,
state_c, *params)
else:
outputs, state_h = rnn(get_inputs(data), state_h, *params)
# 设t_ib_j为i时间批量中的j元素
# label 尺寸：（batch_size * num_steps）
# label = [t_0b_0, t_0b_1, ..., t_1b_0, t_1b_1, ..., ]
label = label.T.reshape((-1, ))
# 拼接outputs，尺寸：(batch_size * num_steps, vocab_size)。
outputs = nd.concat(*outputs, dim=0)
# 经上述操作，outputs和label已对齐。
loss = softmax_cross_entropy(outputs, label)
loss.backward()
grad_clipping(params, clipping_theta, ctx)
utils.SGD(params, learning_rate)
train_loss += nd.sum(loss).asscalar()
num_examples += loss.size
if e % pred_period == 0:
try:
ans = exp(train_loss / num_examples)
print("Epoch %d. Perplexity %f" % (e, ans))
except OverflowError:
print("overflow!", train_loss, ';', num_examples)
print('still we are on our way---batch')
ans = float('inf')
#for seq in seqs:
for i in range(len(seqs)):
seq = seqs[i]
try:
ansres = predict_rnn(rnn, seq, pred_len, params,
hidden_dim, ctx, idx_to_char,
char_to_idx, get_inputs, is_lstm)
print('-', seq, ansres)
#ans_collect.append(ansres.split(' ')[0])
ans_collect.append(re.split(r'[;,\s]\s*', ansres)[0])
with open(
'C:/Users/QTC I7-1060/Desktop/ml_hu/data/que_generate.txt',
'w') as f:
f.write(ansres, '\n')
f.close()
except:
print('theme--', seq, 'unmatched!')
#ans_collect.setdefault(seq,ansres)
print()
return ans_collect
# for param in params:
# param[:] = param - lr * param.grad
# print(utils.evaluate_accuracy(test_data, net))
# 训练
epoches = 5
learning_rate = 0.1
for e in range(epoches):
train_losses = 0.
train_acc = 0.
for data, label in train_data:
with ag.record():
output = net(data)
loss = cross_entropy(output, label)
loss.backward()
utils.SGD(params, learning_rate/batch_size) # 将剃度做平均使学习率对batch_size不敏感
train_losses += nd.mean(loss).asscalar()
train_acc += utils.accuracy(output, label)
test_acc = utils.evaluate_accuracy(test_data, net)
print("Epoach: %d, Loss: %f, Train Acc: %f, Test Acc: %f" %
(e, train_losses/len(train_data), train_acc/len(train_data), test_acc))
seq_len = 35
learning_rate = .1
batch_size = 32
softmax_cross_entropy = gluon.loss.SoftmaxCrossEntropyLoss()
for e in range(epochs + 1):
train_loss, num_examples = 0, 0
state = nd.zeros(shape=(batch_size, num_hidden), ctx=ctx)
for data, label in data_iter(batch_size, seq_len, ctx):
with autograd.record():
outputs, state = rnn(get_inputs(data), state)
# reshape label to (batch_size*seq_len, )
# concate outputs to (batch_size*seq_len, vocab_size)
label = label.T.reshape((-1, ))
outputs = nd.concat(*outputs, dim=0)
loss = softmax_cross_entropy(outputs, label)
loss.backward()
grad_clipping(params, 5)
utils.SGD(params, learning_rate)
train_loss += nd.sum(loss).asscalar()
num_examples += loss.size
if e % 20 == 0:
print("Epoch %d. PPL %f" % (e, exp(train_loss / num_examples)))
print(' - ', predict('The Time ', 100))
print(' - ', predict("The Medical Man rose, came to the lamp,", 100),
'\n')
def start_training(args):
model_name = str(args.model)
output_path = str(args.out_path) + '/'
input_path = str(args.in_path) + '/'
epochs = int(args.epochs)
batch_size = int(args.batch_size)
nb_classes = 1
df_train = pd.read_csv(input_path + 'train_masks.csv')
ids_train = df_train['img'].map(lambda s: s.split('.')[0])
# print(ids_train)
ids_train_split, ids_valid_split = train_test_split(ids_train,
test_size=0.2,
random_state=42)
print('Training on {} samples'.format(len(ids_train_split)))
print('Validating on {} samples'.format(len(ids_valid_split)))
model = None
image_size = 256
# batch_size = 24
model_path = 'weights/best_weights.hdf5'
csv_path = output_path + 'train_log.csv'
if model_name == 'unet128':
model = get_unet_128()
image_size = 128
batch_size = 48
model_path = output_path + 'Unet128_{val_loss:.4f}-{val_binary_accuracy:.4f}-{val_dice_loss:.5f}.hdf5'
csv_path = output_path + 'Unet128.csv'
# tested
elif model_name == 'unet256':
model_path = output_path + 'Unet256_{val_loss:.4f}-{val_binary_accuracy:.4f}-{val_dice_loss:.5f}.hdf5'
csv_path = output_path + 'Unet256.csv'
img_tuple = (256, 256, 3)
batch_size = 24
model = get_unet_256(input_shape=img_tuple, num_classes=nb_classes)
model.compile(optimizer=optimizers.SGD(lr=0.01,
decay=1e-6,
momentum=0.9,
nesterov=True),
loss=weighted_loss,
metrics=['binary_accuracy', dice_loss])
callbacks = [
# EarlyStopping(monitor='val_dice_loss',
# patience=8,
# verbose=1,
# min_delta=1e-05,
# mode='max'),
# ReduceLROnPlateau(monitor='val_dice_loss',
# factor=0.333,
# patience=3,
# verbose=1,
# epsilon=1e-05,
# mode='max',
# min_lr=1e-04),
ModelCheckpoint(monitor='val_dice_loss',
filepath=model_path,
save_best_only=True,
save_weights_only=True,
mode='max'),
TensorBoard(log_dir=output_path),
CSVLogger(filename=csv_path, separator=',', append=True),
LearningRateScheduler(unet_lr_scheduler)
]
# elif model_name == 'unet512':
# model = get_unet_512()
# image_size = 512
# batch_size = 12
# model_path = output_path + 'Unet512_{val_loss:.4f}-{val_binary_accuracy:.4f}-{val_dice_loss:.5f}.hdf5'
# csv_path = output_path + 'Unet512.csv'
elif model_name == 'unet1024':
model_path = output_path + 'Unet1024_best_model.hdf5'
csv_path = output_path + 'Unet1024.csv'
img_tuple = (256, 256, 3)
# batch_size = 3
model = get_unet_1024(input_shape=img_tuple, num_classes=nb_classes)
model.compile(optimizer=utils.SGD(lr=1e-02,
decay=1e-6,
momentum=0.9,
nesterov=True,
accumulator=np.ceil(
float(64) / float(batch_size))),
loss=weighted_loss,
metrics=['binary_accuracy', dice_loss])
callbacks = [
EarlyStopping(monitor='val_dice_loss',
patience=8,
verbose=1,
min_delta=1e-05,
mode='max'),
ReduceLROnPlateau(monitor='val_dice_loss',
factor=0.333,
patience=3,
verbose=1,
epsilon=1e-05,
mode='max',
min_lr=1e-04),
ModelCheckpoint(monitor='val_dice_loss',
filepath=model_path,
save_best_only=True,
save_weights_only=True,
mode='max'),
TensorBoard(log_dir=output_path),
CSVLogger(filename=csv_path, separator=',', append=True),
# LearningRateScheduler(unet_lr_scheduler)
]
elif model_name == 'unet':
model_path = output_path + 'Unet_best_model.hdf5'
csv_path = output_path + 'Unet.csv'
img_tuple = (256, 256, 3)
batch_size = 24
model = get_unet(input_shape=img_tuple, num_classes=nb_classes)
model.compile(optimizer=utils.SGD(lr=1e-02,
decay=1e-6,
momentum=0.9,
nesterov=True,
accumulator=1),
loss=weighted_loss,
metrics=['binary_accuracy', dice_loss])
callbacks = [
EarlyStopping(monitor='val_dice_loss',
patience=6,
verbose=1,
min_delta=1e-05,
mode='max'),
ReduceLROnPlateau(monitor='val_dice_loss',
factor=0.333,
patience=3,
verbose=1,
epsilon=1e-05,
mode='max',
min_lr=3e-04),
ModelCheckpoint(monitor='val_dice_loss',
filepath=model_path,
save_best_only=True,
save_weights_only=True,
mode='max'),
TensorBoard(log_dir=output_path),
CSVLogger(filename=csv_path, separator=',', append=True),
# LearningRateScheduler(unet_lr_scheduler)
]
elif model_name == 'modify_unet':
model_path = output_path + 'Modify_Unet_best_model.hdf5'
csv_path = output_path + 'Modify_Unet.csv'
img_tuple = (512, 512, 3)
# batch_size = 3
model = get_modify_unet(input_shape=img_tuple, num_classes=nb_classes)
if args.noweight:
model.compile(optimizer=utils.SGD(
lr=1e-02,
decay=1e-6,
momentum=0.9,
nesterov=True,
accumulator=np.ceil(float(15) / float(batch_size))),
loss=weighted_loss,
metrics=['binary_accuracy', dice_loss])
else:
model.compile(optimizer=utils.Adam(
lr=1e-02, accumulator=np.ceil(float(15) / float(batch_size))),
loss=weighted_loss,
metrics=['binary_accuracy', dice_loss])
callbacks = [
EarlyStopping(monitor='val_dice_loss',
patience=8,
verbose=1,
min_delta=1e-05,
mode='max'),
ReduceLROnPlateau(monitor='val_dice_loss',
factor=0.333,
patience=3,
verbose=1,
epsilon=1e-04,
mode='max',
min_lr=3e-04),
ModelCheckpoint(monitor='val_dice_loss',
filepath=model_path,
save_best_only=True,
save_weights_only=True,
mode='max'),
TensorBoard(log_dir=output_path),
CSVLogger(filename=csv_path, separator=',', append=True),
# LearningRateScheduler(enet_lr_scheduler)
]
elif model_name == 'modify_unet_V2':
model_path = output_path + 'Modify_Unet_best_model.hdf5'
csv_path = output_path + 'Modify_Unet.csv'
img_tuple = (1024, 1024, 3)
# batch_size = 5
model = get_modify_unet_V2(input_shape=img_tuple,
num_classes=nb_classes)
if args.noweight:
model.compile(optimizer=utils.SGD(
lr=1e-02,
decay=1e-6,
momentum=0.9,
nesterov=True,
accumulator=np.ceil(float(40) / float(batch_size))),
loss=weighted_loss,
metrics=['binary_accuracy', dice_loss])
else:
model.compile(optimizer=utils.SGD(
lr=1e-02,
decay=1e-6,
momentum=0.9,
nesterov=True,
accumulator=np.ceil(float(40) / float(batch_size))),
loss=weighted_loss,
metrics=['binary_accuracy', dice_loss])
callbacks = [
EarlyStopping(monitor='val_dice_loss',
patience=8,
verbose=1,
min_delta=1e-05,
mode='max'),
ReduceLROnPlateau(monitor='val_dice_loss',
factor=0.333,
patience=3,
verbose=1,
epsilon=1e-04,
mode='max',
min_lr=1e-04),
ModelCheckpoint(monitor='val_dice_loss',
filepath=model_path,
save_best_only=True,
save_weights_only=True,
mode='max'),
TensorBoard(log_dir=output_path),
CSVLogger(filename=csv_path, separator=',', append=True),
# LearningRateScheduler(unet_lr_scheduler)
]
# elif model_name == 'segnet':
# model_path = output_path + 'SegNet_{val_loss:.4f}-{val_binary_accuracy:.4f}-{val_dice_loss:.5f}.hdf5'
# csv_path = output_path + 'SegNet.csv'
# image_size = 1024
# batch_size = 1
# model = SegNet(input_shape=(image_size, image_size, 3), classes=nb_classes)
# model.compile(optimizer=optimizers.Adam(lr=0.01), loss=bce_dice_loss, metrics=['binary_accuracy',
# dice_loss])
#
# elif model_name == 'segnet_full':
# model_path = output_path + 'SegNet_FULL_{val_loss:.4f}-{val_binary_accuracy:.4f}-{val_dice_loss:.5f}.hdf5'
# csv_path = output_path + 'SegNet_FULL.csv'
# image_size = 1024
# batch_size = 2
# model = SegNet(input_shape=(image_size, image_size, 3), classes=nb_classes)
# model.compile(optimizer=optimizers.Adam(lr=0.01), loss=bce_dice_loss, metrics=['binary_accuracy',
# dice_loss])
# elif model_name == 'enet_unpooling':
# model_path = output_path + 'Enet_unpooling_{val_loss:.4f}-{val_binary_accuracy:.4f}.hdf5'
# csv_path = output_path + 'Enet_unpooling.csv'
# image_size = int(256)
# model = get_enet_unpooling(input_shape=(image_size, image_size, 3), classes=nb_classes)
# model.compile(optimizer=optimizers.Adam(lr=0.01), loss=losses.binary_crossentropy, metrics=['binary_accuracy',
# dice_loss])
elif model_name == 'enet_naive_upsampling':
model_path = output_path + 'Enet_naive_upsampling_best_model.hdf5'
csv_path = output_path + 'Enet_naive_upsampling.csv'
img_tuple = (512, 512, 3)
# batch_size = 6
model = get_enet_naive_upsampling(input_shape=img_tuple,
classes=nb_classes)
if args.noweight:
model.compile(optimizer=utils.SGD(
lr=1e-02,
decay=1e-6,
momentum=0.9,
nesterov=True,
accumulator=np.ceil(float(14) / float(batch_size))),
loss=weighted_loss,
metrics=['binary_accuracy', dice_loss])
else:
model.compile(optimizer=optimizers.Adam(lr=1e-02),
loss=weighted_loss,
metrics=['binary_accuracy', dice_loss])
callbacks = [
EarlyStopping(monitor='val_dice_loss',
patience=8,
verbose=1,
min_delta=1e-05,
mode='max'),
ReduceLROnPlateau(monitor='val_dice_loss',
factor=0.333,
patience=3,
verbose=1,
epsilon=1e-05,
mode='max',
min_lr=3e-04),
ModelCheckpoint(monitor='val_dice_loss',
filepath=model_path,
save_best_only=True,
save_weights_only=True,
mode='max'),
TensorBoard(log_dir=output_path),
CSVLogger(filename=csv_path, separator=',', append=True),
# LearningRateScheduler(enet_lr_scheduler)
]
# elif model_name == 'densenet_fc':
# model_path = output_path + 'DenseNet_FC_{val_loss:.4f}-{val_binary_accuracy:.4f}-{val_dice_loss:.5f}.hdf5'
# csv_path = output_path + 'DenseNet_FC.csv'
# image_size = 256
# batch_size = 8
# base_model = DenseNetFCN(input_shape=(image_size, image_size, 3), classes=nb_classes, activation='sigmoid', include_top=False, batchsize=batch_size)
# classify = Conv2D(nb_classes, (1, 1), activation='sigmoid')(base_model.output)
# model = Model(inputs=base_model.input, outputs=classify)
# model.compile(optimizer=optimizers.Adam(lr=0.01), loss=bce_dice_loss, metrics=['binary_accuracy',
# dice_loss])
elif model_name == 'linknet':
model_path = output_path + 'LinkNet_best_model.hdf5'
csv_path = output_path + 'LinkNet.csv'
img_tuple = (1024, 1024, 3)
batch_size = 6
base_model = LinkNet(input_shape=img_tuple, classes=nb_classes)
classify = Conv2D(nb_classes, (1, 1),
activation='sigmoid')(base_model.output)
model = Model(inputs=base_model.input, outputs=classify)
model.compile(optimizer=utils.SGD(lr=1e-02,
decay=1e-6,
momentum=0.9,
nesterov=True,
accumulator=4),
loss=weighted_loss,
metrics=['binary_accuracy', dice_loss])
callbacks = [
EarlyStopping(monitor='val_dice_loss',
patience=8,
verbose=1,
min_delta=1e-05,
mode='max'),
ReduceLROnPlateau(monitor='val_dice_loss',
factor=0.333,
patience=3,
verbose=1,
epsilon=1e-05,
mode='max',
min_lr=1e-04),
ModelCheckpoint(monitor='val_dice_loss',
filepath=model_path,
save_best_only=True,
save_weights_only=True,
mode='max'),
TensorBoard(log_dir=output_path),
CSVLogger(filename=csv_path, separator=',', append=True),
# LearningRateScheduler(linknet_lr_scheduler)
]
elif model_name == 'linknet_res34':
model_path = output_path + 'LinkNet_Res34_best_model.hdf5'
csv_path = output_path + 'LinkNet_Res34.csv'
img_tuple = (1024, 1024, 3)
# batch_size = 12
base_model = LinkNet_Res34(input_shape=img_tuple, classes=nb_classes)
classify = Conv2D(nb_classes, (1, 1),
activation='sigmoid')(base_model.output)
model = Model(inputs=base_model.input, outputs=classify)
model.compile(optimizer=utils.SGD(lr=1e-02,
decay=1e-6,
momentum=0.9,
nesterov=True,
accumulator=np.ceil(
float(84) / float(batch_size))),
loss=weighted_loss_256,
metrics=['binary_accuracy', dice_loss])
callbacks = [
EarlyStopping(monitor='val_dice_loss',
patience=8,
verbose=1,
min_delta=1e-05,
mode='max'),
ReduceLROnPlateau(monitor='val_dice_loss',
factor=0.333,
patience=3,
verbose=1,
epsilon=1e-05,
mode='max',
min_lr=1e-04),
ModelCheckpoint(monitor='val_dice_loss',
filepath=model_path,
save_best_only=True,
save_weights_only=True,
mode='max'),
TensorBoard(log_dir=output_path),
CSVLogger(filename=csv_path, separator=',', append=True),
# LearningRateScheduler(unet_lr_scheduler)
]
elif model_name == 'linknet_res50':
model_path = output_path + 'LinkNet_Res50_{val_loss:.4f}-{val_binary_accuracy:.4f}-{val_dice_loss:.5f}.hdf5'
csv_path = output_path + 'LinkNet_Res50.csv'
img_tuple = (1280, 1280, 3)
batch_size = 2
base_model = LinkNet_Res50(input_shape=img_tuple, classes=nb_classes)
classify = Conv2D(nb_classes, (1, 1),
activation='sigmoid')(base_model.output)
model = Model(inputs=base_model.input, outputs=classify)
model.compile(optimizer=utils.SGD(lr=0.01,
decay=1e-6,
momentum=0.9,
nesterov=True,
accumulator=10),
loss=weighted_loss,
metrics=['binary_accuracy', dice_loss])
callbacks = [
# EarlyStopping(monitor='val_dice_loss',
# patience=8,
# verbose=1,
# min_delta=1e-05,
# mode='max'),
# ReduceLROnPlateau(monitor='val_dice_loss',
# factor=0.333,
# patience=3,
# verbose=1,
# epsilon=1e-05,
# mode='max',
# min_lr=1e-04),
ModelCheckpoint(monitor='val_dice_loss',
filepath=model_path,
save_best_only=True,
save_weights_only=True,
mode='max'),
TensorBoard(log_dir=output_path),
CSVLogger(filename=csv_path, separator=',', append=True),
LearningRateScheduler(unet_lr_scheduler)
]
if model:
train_batch_gen = iter(
Batch_Generator(input_path=input_path,
idx=ids_train_split,
batch_size=batch_size,
img_tuple=img_tuple,
aug=True,
sat=args.sat,
hue=args.hue,
val=args.val,
rotate=args.rotate,
gray=args.gray,
contrast=args.contrast,
brightness=args.brightness,
shear=args.shear,
shift=args.shift,
scale=args.scale,
lowq=args.lowq))
valid_batch_gen = iter(
Batch_Generator(input_path=input_path,
idx=ids_valid_split,
batch_size=batch_size,
img_tuple=img_tuple,
aug=False))
print(
'Start to train model hue %r, sat %r, val %r, rotate %r, gray %r, contrast %r, brightness %r, shear %r, shift %r, scale %r, lowq %r, none weight %r'
% (
args.hue,
args.sat,
args.val,
args.rotate,
args.gray,
args.contrast,
args.brightness,
args.shear,
args.shift,
args.scale,
args.lowq,
args.noweight,
))
model.fit_generator(
generator=train_batch_gen,
steps_per_epoch=np.ceil(
float(len(ids_train_split)) / float(batch_size)),
epochs=epochs,
callbacks=callbacks,
validation_data=valid_batch_gen,
validation_steps=np.ceil(
float(len(ids_valid_split)) / float(batch_size)),
workers=6)
h1 = relu(nd.dot(X, W1) + b1)# 隐含层输出 非线性激活
output = nd.dot(h1, W2) + b2
return output
##Softmax和交叉熵损失函数
## softmax 回归实现 exp(Xi)/(sum(exp(Xi))) 归一化概率 使得 10类概率之和为1
#交叉熵损失函数
softmax_cross_entropy = gluon.loss.SoftmaxCrossEntropyLoss()
## 开始训练
learning_rate = .5#学习率
epochs = 7 ##训练迭代训练集 次数
for epoch in range(epochs):##每迭代一次训练集
train_loss = 0.##损失
train_acc = 0. ##准确度
for data, label in train_data:#训练集
with autograd.record():#自动微分
output = net(data)#模型输出 向前传播
loss = softmax_cross_entropy(output, label)#计算损失
loss.backward()#向后传播
utils.SGD(params, learning_rate/batch_size)#随机梯度下降 训练更新参数 学习率递减
train_loss += nd.mean(loss).asscalar()#损失
train_acc += utils.accuracy(output, label)#准确度
test_acc = utils.evaluate_accuracy(test_data, net)#测试集测试
print("E次数 %d. 损失: %f, 训练准确度 %f, 测试准确度%f" % (
epoch, train_loss/len(train_data),
train_acc/len(train_data), test_acc))
def main_worker(gpu, ngpus_per_node, args):
global best_acc1
args.gpu = gpu
if args.gpu is not None:
print("Use GPU: {} for training".format(args.gpu))
if args.distributed:
if args.dist_url == "env://" and args.rank == -1:
args.rank = int(os.environ["RANK"])
if args.multiprocessing_distributed:
# For multiprocessing distributed training, rank needs to be the
# global rank among all the processes
args.rank = args.rank * ngpus_per_node + gpu
dist.init_process_group(backend=args.dist_backend,
init_method=args.dist_url,
world_size=args.world_size,
rank=args.rank)
# create model
if args.pretrained:
print("=> using pre-trained model '{}'".format(args.arch))
# model = models.__dict__[args.arch](pretrained=True)
model = resnet.__dict__[args.arch](pretrained=True)
else:
print("=> creating model '{}'".format(args.arch))
# model = models.__dict__[args.arch]()
model = resnet.__dict__[args.arch]()
if args.distributed:
# For multiprocessing distributed, DistributedDataParallel constructor
# should always set the single device scope, otherwise,
# DistributedDataParallel will use all available devices.
if args.gpu is not None:
torch.cuda.set_device(args.gpu)
model.cuda(args.gpu)
# When using a single GPU per process and per
# DistributedDataParallel, we need to divide the batch size
# ourselves based on the total number of GPUs we have
args.batch_size = int(args.batch_size / ngpus_per_node)
args.workers = int(
(args.workers + ngpus_per_node - 1) / ngpus_per_node)
model = torch.nn.parallel.DistributedDataParallel(
model, device_ids=[args.gpu])
else:
model.cuda()
# DistributedDataParallel will divide and allocate batch_size to all
# available GPUs if device_ids are not set
model = torch.nn.parallel.DistributedDataParallel(model)
elif args.gpu is not None:
torch.cuda.set_device(args.gpu)
model = model.cuda(args.gpu)
else:
# DataParallel will divide and allocate batch_size to all available GPUs
if args.arch.startswith('alexnet') or args.arch.startswith('vgg'):
model.features = torch.nn.DataParallel(model.features)
model.cuda()
else:
model = torch.nn.DataParallel(model).cuda()
# define loss function (criterion) and optimizer
criterion = nn.CrossEntropyLoss().cuda(args.gpu)
optimizer = utils.SGD(model.parameters(),
args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
# optionally resume from a checkpoint
if args.resume:
if os.path.isfile(args.resume):
print("=> loading checkpoint '{}'".format(args.resume))
checkpoint = torch.load(args.resume)
args.start_epoch = checkpoint['epoch']
best_acc1 = checkpoint['best_acc1']
if args.gpu is not None:
# best_acc1 may be from a checkpoint from a different GPU
best_acc1 = best_acc1.to(args.gpu)
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
print("=> loaded checkpoint '{}' (epoch {})".format(
args.resume, checkpoint['epoch']))
else:
print("=> no checkpoint found at '{}'".format(args.resume))
cudnn.benchmark = True
# Data loading code
traindir = os.path.join(args.data, 'train')
valdir = os.path.join(args.data, 'val')
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
train_dataset = datasets.ImageFolder(
traindir,
transforms.Compose([
transforms.RandomResizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
normalize,
]))
if args.distributed:
train_sampler = torch.utils.data.distributed.DistributedSampler(
train_dataset)
else:
train_sampler = None
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=(train_sampler is None),
num_workers=args.workers,
pin_memory=True,
sampler=train_sampler)
val_loader = torch.utils.data.DataLoader(datasets.ImageFolder(
valdir,
transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
normalize,
])),
batch_size=args.batch_size,
shuffle=False,
num_workers=args.workers,
pin_memory=True)
if args.evaluate:
validate(val_loader, model, criterion, args)
return
#*quantize weights before training:
for name, param in model.named_parameters():
print(name)
if name.find('conv') > -1:
param.data = qu(param.data)
print("Weights quantized...")
for epoch in range(args.start_epoch, args.epochs):
if args.distributed:
train_sampler.set_epoch(epoch)
adjust_learning_rate(optimizer, epoch, args)
# train for one epoch
train(train_loader, model, criterion, optimizer, epoch, args)
# evaluate on validation set
acc1 = validate(val_loader, model, criterion, args)
# remember best acc@1 and save checkpoint
is_best = acc1 > best_acc1
best_acc1 = max(acc1, best_acc1)
if not args.multiprocessing_distributed or (
args.multiprocessing_distributed
and args.rank % ngpus_per_node == 0):
save_checkpoint(
{
'epoch': epoch + 1,
'arch': args.arch,
'state_dict': model.state_dict(),
'best_acc1': best_acc1,
'optimizer': optimizer.state_dict(),
}, is_best)
train_data = copy.deepcopy(raw_data)
dup_rawData = copy.deepcopy(train_data)
ground_truth = train_data[:, 57] #(4001, 1)
train_data = np.delete(train_data, -1, 1)
print("dim of raw_data line 23", np.shape(train_data))
################# training process, SGD #################
w = []
b = []
epochs = 300
learning_rate = 1e-5 # best so far (100, 1e-5), (300, 1000)
w, b = utils.SGD(train_data, ground_truth, learning_rate, epochs)
################# training process, mini-batch SG #################
'''
w = []
b = []
epochs = 10000
learning_rate = 1e-7 # best so far (4000, 1e-7),(1000, 1e-7),
w,b = utils.batchGD(train_data, ground_truth, 1000, learning_rate, epochs)
'''
# open test data
test_data = []
