def train(file_train, file_test, epochs, batch_size, input_dim_size, hidden_dim_size, output_dim_size, learning_rate): model = NeuralNetwork(input_dim_size, hidden_dim_size, output_dim_size) optimizer = utils.SGD(model, learning_rate) train_loader = load.DataLoader(file_train) train_features, train_labels = train_loader.load() for epoch in range(epochs): for minibatch_features, minibatch_labels in utils.get_batches( train_features, train_labels, batch_size, shuffle=True): # 順伝播 minibatch_features_reshaped = minibatch_features.T z1, minibatch_predicted_labels = model.forward( minibatch_features_reshaped) # 逆伝播 grads = model.backward(x=minibatch_features, z1=z1, y=minibatch_predicted_labels, t=minibatch_labels) # パラメータの更新 optimizer.update(grads) # テストデータによる Inference と評価 accuracy = inference_test.infer(file_test=file_test, model_trained=model) print('[{}] EPOCH {} Accuracy:{:.8f}'.format(datetime.datetime.today(), epoch, accuracy)) print('[{}] Finished Training'.format(datetime.datetime.today())) return model
def train(): batch_size = 256 train_data, test_data = utils.load_data_fashion_mnist(batch_size) softmax_cross_entropy = gluon.loss.SoftmaxCrossEntropyLoss() learning_rate = 0.2 for epoch in range(5): train_loss = 0. train_acc = 0. for data, label in train_data: label = label.as_in_context(ctx) with autograd.record(): output = net(data, is_training=True) loss = softmax_cross_entropy(output, label) loss.backward() utils.SGD(params, learning_rate / batch_size) train_loss += nd.mean(loss).asscalar() train_acc += utils.accuracy(output, label) test_acc = utils.evaluate_accuracy(test_data, net, ctx) print("Epoch %d. Loss: %f, Train acc %f, Test acc %f" % (epoch, train_loss / len(train_data), train_acc / len(train_data), test_acc))
def train_and_predict_rnn(rnn, is_random_iter, epochs, num_steps, hidden_dim, learning_rate, clipping_theta, batch_size, pred_period, pred_len, seqs, get_params, get_inputs, ctx, corpus_indices, idx_to_char, char_to_idx, is_lstm=False): if is_random_iter: data_iter = data_iter_random else: data_iter = data_iter_consecutive params = get_params() softmax_cross_entropy = gluon.loss.SoftmaxCrossEntropyLoss() for e in range(1, epochs+1): # 如用相邻批量采样,在同一个 epoch 中,隐含变量只需要在该 epoch 开始的时候初始化。 if not is_random_iter: state_h = nd.zeros(shape=(batch_size, hidden_dim), ctx=ctx) if is_lstm: # 当 RNN 使用LSTM时才会用到,这里可以忽略 state_c = nd.zeros(shape=(batch_size, hidden_dim), ctx=ctx) train_loss, num_examples = 0, 0 for data, label in data_iter(corpus_indices, batch_size, num_steps, ctx): # 如使用随机批量采样,处理每个随机小批量前都需要初始化隐含变量。 if is_random_iter: state_h = nd.zeros(shape=(batch_size, hidden_dim), ctx=ctx) if is_lstm: # 当 RNN 使用 LSTM 时才会用到,这里可以忽略。 state_c = nd.zeros(shape=(batch_size, hidden_dim), ctx=ctx) with autograd.record(): # outputs 尺寸: (batch_size, vocab_size) if is_lstm: # 当RNN 使用 LSTM 时才会用到,这里可以忽略。 outputs, state_h, state_c = rnn( get_inputs(data), state_h, state_c, *params) else: outputs, state_h = rnn(get_inputs(data), state_h, *params) # 设 t_ib_j 为 i 时间批量中的 j 元素: # label 尺寸: (batch_size * num_steps) # label = [t_0b_0, t_0b_1, ... , t_1b_0, t_1b_1, ..., ] label = label.T.reshape((-1,)) # 拼接 outputs, 尺寸: (batch_size * num_steps, vocab_size) outputs = nd.concat(*outputs, dim=0) # 经上述操作,outputs 和 label 已对齐 loss = softmax_cross_entropy(outputs, label) loss.backward() grad_clipping(params, clipping_theta, ctx) utils.SGD(params, learning_rate) train_loss += nd.sum(loss).asscalar() num_examples += loss.size if e % pred_period == 0: print("Epoch %d. Perplexity %f" % (e, exp(train_loss/num_examples))) for seq in seqs: print(' - ', predict_rnn(rnn, seq, pred_len, params, hidden_dim, ctx, idx_to_char, char_to_idx, get_inputs, is_lstm)) print()
def train_batch(data, label, params, ctx, lr): data_list = split_and_load(data, ctx) label_list = split_and_load(label, ctx) with autograd.record(): losses = [loss(lenet(X, W), Y) for X, W, Y in zip(data_list, params, label_list)] for l in losses: l.backward() for i in range(len(params[0])): allreduce(params[c][i].grad for c in range(len(ctx))) for p in params: utils.SGD(p, lr / data.shape[0])
def train(): for epoch in range(5): train_loss = 0. train_acc = 0. for data, label in train_data: with autograd.record(): output = net(data) loss = softmax_cross_entropy(output, label) loss.backward() utils.SGD(params, learning_rate / batch_size) train_loss += nd.mean(loss).asscalar() train_acc += utils.accuracy(output, label) test_acc = utils.evaluate_accuracy(test_data, net) print("Epoch %d. Loss: %f, Train acc %f, Test acc %f" % (epoch, train_loss / len(train_data), train_acc / len(train_data), test_acc))
h2 = dropout(h2,drop_prob2) return nd.dot(h2,W3)+b3 from mxnet import autograd from mxnet import gluon softmax_cross_entropy = gluon.loss.SoftmaxCrossEntropyLoss() learning_rate = .5 epochs = 10 for epoch in range(epochs): train_loss = 0. train_acc = 0. for data,label in train_data: with autograd.record(): output = net(data) loss = softmax_cross_entropy(output,label) loss.backward() utils.SGD(params,learning_rate/batch_size) train_loss += nd.mean(loss).asscalar() train_acc +=utils.accuracy(output,label) test_acc = utils.evaluate_accuracy(test_data,net) print("Epoch: %d. train loss: %f, train acc: %f, test acc: %f"%(epoch,train_loss/len(train_data),train_acc/len(train_data),test_acc))
def train_and_predict_rnn(rnn, is_random_iter, epochs, num_steps, hidden_dim, learning_rate, clipping_norm, batch_size, pred_period, pred_len, seqs, get_params, get_inputs, ctx, corpus_indices, idx_to_char, char_to_idx, is_lstm=False): """Train an RNN model and predict the next item in the sequence.""" if is_random_iter: data_iter = data_iter_random else: data_iter = data_iter_consecutive params = get_params() softmax_cross_entropy = gluon.loss.SoftmaxCrossEntropyLoss() for e in range(1, epochs + 1): # If consecutive sampling is used, in the same epoch, the hidden state # is initialized only at the beginning of the epoch. if not is_random_iter: state_h = nd.zeros(shape=(batch_size, hidden_dim), ctx=ctx) if is_lstm: state_c = nd.zeros(shape=(batch_size, hidden_dim), ctx=ctx) train_loss, num_examples = 0, 0 for data, label in data_iter(corpus_indices, batch_size, num_steps, ctx): # If random sampling is used, the hidden state has to be # initialized for each mini-batch. if is_random_iter: state_h = nd.zeros(shape=(batch_size, hidden_dim), ctx=ctx) if is_lstm: state_c = nd.zeros(shape=(batch_size, hidden_dim), ctx=ctx) with autograd.record(): # outputs shape: (batch_size, vocab_size) if is_lstm: outputs, state_h, state_c = rnn(get_inputs(data), state_h, state_c, *params) else: outputs, state_h = rnn(get_inputs(data), state_h, *params) # Let t_ib_j be the j-th element of the mini-batch at time i. # label shape: (batch_size * num_steps) # label = [t_0b_0, t_0b_1, ..., t_1b_0, t_1b_1, ..., ]. label = label.T.reshape((-1,)) # Concatenate outputs: # shape: (batch_size * num_steps, vocab_size). outputs = nd.concat(*outputs, dim=0) # Now outputs and label are aligned. loss = softmax_cross_entropy(outputs, label) loss.backward() grad_clipping(params, clipping_norm, ctx) utils.SGD(params, learning_rate) train_loss += nd.sum(loss).asscalar() num_examples += loss.size if e % pred_period == 0: print("Epoch %d. Training perplexity %f" % (e, exp(train_loss/num_examples))) for seq in seqs: print(' - ', predict_rnn(rnn, seq, pred_len, params, hidden_dim, ctx, idx_to_char, char_to_idx, get_inputs, is_lstm)) print()
def predict(args): nb_classes = 1 orig_width = 1918 orig_height = 1280 model_name = str(args.model) model_weight = str(args.model_weight) output_path = str(args.out_path) input_path = str(args.in_path) batch_size = int(args.batchsize) image_size = int(args.image_size) df_test = pd.read_csv(input_path + '/sample_submission.csv') ids_test = df_test['img'].map(lambda s: s.split('.')[0]) image_size = 256 threshold = 0.5 if model_name == 'unet128': image_size = 128 batch_size = 48 model = get_unet_128() model.load_weights(filepath=model_weight) # tested # elif model_name == 'unet256': # image_size = 256 # batch_size = 24 # model = get_unet_256() # model.load_weights(filepath=model_weight) # elif model_name == 'unet512': # image_size = 512 # batch_size = 12 # model = get_unet_512() # model.load_weights(filepath=model_weight) elif model_name == 'unet1024': img_tuple = (256, 256, 3) batch_size = 64 model = get_unet_1024(input_shape=img_tuple, num_classes=nb_classes) model.compile(optimizer=utils.SGD(lr=1e-02, decay=1e-6, momentum=0.9, nesterov=True, accumulator=np.ceil( float(20) / float(batch_size))), loss=weighted_loss, metrics=['binary_accuracy', dice_loss]) model.load_weights(filepath=model_weight) # elif model_name == 'segnet': # image_size = 256 # batch_size = 28 # model = SegNet(input_shape=(image_size, image_size, 3), classes=nb_classes) # model.compile(optimizer=optimizers.Adam(lr=0.01), loss=bce_dice_loss, metrics=['binary_accuracy', # dice_loss]) # # model.load_weights(filepath=model_weight) # elif model_name == 'enet_unpooling': # model_path = output_path + 'Enet_unpooling_{val_loss:.4f}-{val_binary_accuracy:.4f}.hdf5' # csv_path = output_path + 'Enet_unpooling.csv' # image_size = int(256) # model = get_enet_unpooling(input_shape=(image_size, image_size, 3), classes=nb_classes) # model.compile(optimizer=optimizers.Adam(lr=0.01), loss=losses.binary_crossentropy, metrics=['binary_accuracy', # dice_loss]) elif model_name == 'enet_naive_upsampling': img_tuple = (512, 512, 3) batch_size = 4 model = get_enet_naive_upsampling(input_shape=img_tuple, classes=nb_classes) model.compile(optimizer=utils.SGD(lr=1e-02, decay=1e-6, momentum=0.9, nesterov=True, accumulator=5), loss=weighted_loss, metrics=['binary_accuracy', dice_loss]) model.load_weights(filepath=model_weight) # elif model_name == 'init_unet': # img_tuple = (1024, 1024, 3) # batch_size = 6 # model = get_init_unet(input_shape=img_tuple, num_classes=nb_classes) # model.compile(optimizer=utils.SGD(lr=1e-02, decay=1e-6, momentum=0.9, nesterov=True, accumulator=5), # loss=weighted_loss, # metrics=['binary_accuracy', dice_loss]) # model.load_weights(filepath=model_weight) # # elif model_name == 'unet': # img_tuple = (256, 256, 3) # batch_size = 24 # model = get_unet(input_shape=img_tuple, num_classes=nb_classes) # model.compile(optimizer=utils.SGD(lr=1e-02, decay=1e-6, momentum=0.9, nesterov=True, accumulator=5), # loss=weighted_loss, # metrics=['binary_accuracy', dice_loss]) # model.load_weights(filepath=model_weight) elif model_name == 'modify_unet': img_tuple = (512, 512, 3) batch_size = 5 model = get_modify_unet(input_shape=img_tuple, num_classes=nb_classes) model.compile(optimizer=utils.SGD(lr=1e-02, decay=1e-6, momentum=0.9, nesterov=True, accumulator=5), loss=weighted_loss, metrics=['binary_accuracy', dice_loss]) model.load_weights(filepath=model_weight) # elif model_name == 'modify_unet_V2': # img_tuple = (1024, 1024, 3) # batch_size = 4 # model = get_modify_unet_V2(input_shape=img_tuple, num_classes=nb_classes) # model.compile(optimizer=utils.SGD(lr=1e-02, decay=1e-6, momentum=0.9, nesterov=True, accumulator=5), # loss=weighted_loss, # metrics=['binary_accuracy', dice_loss]) # model.load_weights(filepath=model_weight) # # elif model_name == 'linknet': # img_tuple = (1024, 1024, 3) # batch_size = 6 # # base_model = LinkNet(input_shape=img_tuple, classes=nb_classes) # classify = Conv2D(nb_classes, (1, 1), activation='sigmoid')(base_model.output) # model = Model(inputs=base_model.input, outputs=classify) # model.compile(optimizer=utils.SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True, accumulator=10), # loss=weighted_loss, # metrics=['binary_accuracy', dice_loss]) # # model.load_weights(filepath=model_weight) elif model_name == 'linknet_res34': img_tuple = (256, 256, 3) batch_size = 84 base_model = LinkNet_Res34(input_shape=img_tuple, classes=nb_classes) classify = Conv2D(nb_classes, (1, 1), activation='sigmoid')(base_model.output) model = Model(inputs=base_model.input, outputs=classify) model.compile(optimizer=utils.SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True, accumulator=10), loss=weighted_loss, metrics=['binary_accuracy', dice_loss]) model.load_weights(filepath=model_weight) # elif model_name == 'linknet_res50': # img_tuple = (1280, 1280, 3) # batch_size = 4 # # base_model = LinkNet_Res50(input_shape=img_tuple, classes=nb_classes) # classify = Conv2D(nb_classes, (1, 1), activation='sigmoid')(base_model.output) # model = Model(inputs=base_model.input, outputs=classify) # model.compile(optimizer=utils.SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True, accumulator=10), # loss=weighted_loss, # metrics=['binary_accuracy', dice_loss]) # # model.load_weights(filepath=model_weight) names = [] for id in ids_test: names.append('{}.jpg'.format(id)) rles = [] print('Predicting on {} samples with batch_size = {}...'.format( len(ids_test), batch_size)) print('Conditions low quality %r' % (args.lowq, )) for start in tqdm(range(0, len(ids_test), batch_size)): x_batch = [] end = min(start + batch_size, len(ids_test)) ids_test_batch = ids_test[start:end] for id in ids_test_batch.values: if args.lowq: img = cv2.imread(input_path + 'test/{}.jpg'.format(id)) else: img = cv2.imread(input_path + 'test_hq/{}.jpg'.format(id)) img = cv2.resize(img, (img_tuple[1], img_tuple[0])) x_batch.append(img) x_batch = np.array(x_batch, np.float32) / 255 preds = model.predict_on_batch(x_batch) preds = np.squeeze(preds, axis=3) for pred in preds: prob = cv2.resize(pred, (orig_width, orig_height)) mask = prob > threshold rle = run_length_encode(mask) rles.append(rle) print("Generating submission file...") df = pd.DataFrame({'img': names, 'rle_mask': rles}) df.to_csv(output_path + model_weight.split('/')[-1] + '.csv.gz', index=False, compression='gzip')
def train_and_predict_rnn(rnn, is_random_iter, epochs, num_steps, hidden_dim, learning_rate, clipping_theta, batch_size, pred_period, pred_len, seqs, get_params, get_inputs, ctx, corpus_indices, idx_to_char, char_to_idx, is_lstm=False): if is_random_iter: data_iter = data_iter_random else: data_iter = data_iter_consecutive params = get_params() softmax_cross_entropy = gluon.loss.SoftmaxCrossEntropyLoss() ans_collect = [] for e in range(1, epochs + 1): # 如使用相邻批量采样,在同一个epoch中,隐含变量只需要在该epoch开始的时候初始化。 if not is_random_iter: state_h = nd.zeros(shape=(batch_size, hidden_dim), ctx=ctx) if is_lstm: # 当RNN使用LSTM时才会用到,这里可以忽略。 state_c = nd.zeros(shape=(batch_size, hidden_dim), ctx=ctx) train_loss, num_examples = 0, 0 for data, label in data_iter(corpus_indices, batch_size, num_steps, ctx): # 如使用随机批量采样,处理每个随机小批量前都需要初始化隐含变量。 if is_random_iter: state_h = nd.zeros(shape=(batch_size, hidden_dim), ctx=ctx) if is_lstm: # 当RNN使用LSTM时才会用到,这里可以忽略。 state_c = nd.zeros(shape=(batch_size, hidden_dim), ctx=ctx) with autograd.record(): # outputs 尺寸:(batch_size, vocab_size) if is_lstm: # 当RNN使用LSTM时才会用到,这里可以忽略。 outputs, state_h, state_c = rnn(get_inputs(data), state_h, state_c, *params) else: outputs, state_h = rnn(get_inputs(data), state_h, *params) # 设t_ib_j为i时间批量中的j元素 # label 尺寸:(batch_size * num_steps) # label = [t_0b_0, t_0b_1, ..., t_1b_0, t_1b_1, ..., ] label = label.T.reshape((-1, )) # 拼接outputs,尺寸:(batch_size * num_steps, vocab_size)。 outputs = nd.concat(*outputs, dim=0) # 经上述操作,outputs和label已对齐。 loss = softmax_cross_entropy(outputs, label) loss.backward() grad_clipping(params, clipping_theta, ctx) utils.SGD(params, learning_rate) train_loss += nd.sum(loss).asscalar() num_examples += loss.size if e % pred_period == 0: try: ans = exp(train_loss / num_examples) print("Epoch %d. Perplexity %f" % (e, ans)) except OverflowError: print("overflow!", train_loss, ';', num_examples) print('still we are on our way---batch') ans = float('inf') #for seq in seqs: for i in range(len(seqs)): seq = seqs[i] try: ansres = predict_rnn(rnn, seq, pred_len, params, hidden_dim, ctx, idx_to_char, char_to_idx, get_inputs, is_lstm) print('-', seq, ansres) #ans_collect.append(ansres.split(' ')[0]) ans_collect.append(re.split(r'[;,\s]\s*', ansres)[0]) with open( 'C:/Users/QTC I7-1060/Desktop/ml_hu/data/que_generate.txt', 'w') as f: f.write(ansres, '\n') f.close() except: print('theme--', seq, 'unmatched!') #ans_collect.setdefault(seq,ansres) print() return ans_collect
# for param in params: # param[:] = param - lr * param.grad # print(utils.evaluate_accuracy(test_data, net)) # 训练 epoches = 5 learning_rate = 0.1 for e in range(epoches): train_losses = 0. train_acc = 0. for data, label in train_data: with ag.record(): output = net(data) loss = cross_entropy(output, label) loss.backward() utils.SGD(params, learning_rate/batch_size) # 将剃度做平均使学习率对batch_size不敏感 train_losses += nd.mean(loss).asscalar() train_acc += utils.accuracy(output, label) test_acc = utils.evaluate_accuracy(test_data, net) print("Epoach: %d, Loss: %f, Train Acc: %f, Test Acc: %f" % (e, train_losses/len(train_data), train_acc/len(train_data), test_acc))
seq_len = 35 learning_rate = .1 batch_size = 32 softmax_cross_entropy = gluon.loss.SoftmaxCrossEntropyLoss() for e in range(epochs + 1): train_loss, num_examples = 0, 0 state = nd.zeros(shape=(batch_size, num_hidden), ctx=ctx) for data, label in data_iter(batch_size, seq_len, ctx): with autograd.record(): outputs, state = rnn(get_inputs(data), state) # reshape label to (batch_size*seq_len, ) # concate outputs to (batch_size*seq_len, vocab_size) label = label.T.reshape((-1, )) outputs = nd.concat(*outputs, dim=0) loss = softmax_cross_entropy(outputs, label) loss.backward() grad_clipping(params, 5) utils.SGD(params, learning_rate) train_loss += nd.sum(loss).asscalar() num_examples += loss.size if e % 20 == 0: print("Epoch %d. PPL %f" % (e, exp(train_loss / num_examples))) print(' - ', predict('The Time ', 100)) print(' - ', predict("The Medical Man rose, came to the lamp,", 100), '\n')
def start_training(args): model_name = str(args.model) output_path = str(args.out_path) + '/' input_path = str(args.in_path) + '/' epochs = int(args.epochs) batch_size = int(args.batch_size) nb_classes = 1 df_train = pd.read_csv(input_path + 'train_masks.csv') ids_train = df_train['img'].map(lambda s: s.split('.')[0]) # print(ids_train) ids_train_split, ids_valid_split = train_test_split(ids_train, test_size=0.2, random_state=42) print('Training on {} samples'.format(len(ids_train_split))) print('Validating on {} samples'.format(len(ids_valid_split))) model = None image_size = 256 # batch_size = 24 model_path = 'weights/best_weights.hdf5' csv_path = output_path + 'train_log.csv' if model_name == 'unet128': model = get_unet_128() image_size = 128 batch_size = 48 model_path = output_path + 'Unet128_{val_loss:.4f}-{val_binary_accuracy:.4f}-{val_dice_loss:.5f}.hdf5' csv_path = output_path + 'Unet128.csv' # tested elif model_name == 'unet256': model_path = output_path + 'Unet256_{val_loss:.4f}-{val_binary_accuracy:.4f}-{val_dice_loss:.5f}.hdf5' csv_path = output_path + 'Unet256.csv' img_tuple = (256, 256, 3) batch_size = 24 model = get_unet_256(input_shape=img_tuple, num_classes=nb_classes) model.compile(optimizer=optimizers.SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True), loss=weighted_loss, metrics=['binary_accuracy', dice_loss]) callbacks = [ # EarlyStopping(monitor='val_dice_loss', # patience=8, # verbose=1, # min_delta=1e-05, # mode='max'), # ReduceLROnPlateau(monitor='val_dice_loss', # factor=0.333, # patience=3, # verbose=1, # epsilon=1e-05, # mode='max', # min_lr=1e-04), ModelCheckpoint(monitor='val_dice_loss', filepath=model_path, save_best_only=True, save_weights_only=True, mode='max'), TensorBoard(log_dir=output_path), CSVLogger(filename=csv_path, separator=',', append=True), LearningRateScheduler(unet_lr_scheduler) ] # elif model_name == 'unet512': # model = get_unet_512() # image_size = 512 # batch_size = 12 # model_path = output_path + 'Unet512_{val_loss:.4f}-{val_binary_accuracy:.4f}-{val_dice_loss:.5f}.hdf5' # csv_path = output_path + 'Unet512.csv' elif model_name == 'unet1024': model_path = output_path + 'Unet1024_best_model.hdf5' csv_path = output_path + 'Unet1024.csv' img_tuple = (256, 256, 3) # batch_size = 3 model = get_unet_1024(input_shape=img_tuple, num_classes=nb_classes) model.compile(optimizer=utils.SGD(lr=1e-02, decay=1e-6, momentum=0.9, nesterov=True, accumulator=np.ceil( float(64) / float(batch_size))), loss=weighted_loss, metrics=['binary_accuracy', dice_loss]) callbacks = [ EarlyStopping(monitor='val_dice_loss', patience=8, verbose=1, min_delta=1e-05, mode='max'), ReduceLROnPlateau(monitor='val_dice_loss', factor=0.333, patience=3, verbose=1, epsilon=1e-05, mode='max', min_lr=1e-04), ModelCheckpoint(monitor='val_dice_loss', filepath=model_path, save_best_only=True, save_weights_only=True, mode='max'), TensorBoard(log_dir=output_path), CSVLogger(filename=csv_path, separator=',', append=True), # LearningRateScheduler(unet_lr_scheduler) ] elif model_name == 'unet': model_path = output_path + 'Unet_best_model.hdf5' csv_path = output_path + 'Unet.csv' img_tuple = (256, 256, 3) batch_size = 24 model = get_unet(input_shape=img_tuple, num_classes=nb_classes) model.compile(optimizer=utils.SGD(lr=1e-02, decay=1e-6, momentum=0.9, nesterov=True, accumulator=1), loss=weighted_loss, metrics=['binary_accuracy', dice_loss]) callbacks = [ EarlyStopping(monitor='val_dice_loss', patience=6, verbose=1, min_delta=1e-05, mode='max'), ReduceLROnPlateau(monitor='val_dice_loss', factor=0.333, patience=3, verbose=1, epsilon=1e-05, mode='max', min_lr=3e-04), ModelCheckpoint(monitor='val_dice_loss', filepath=model_path, save_best_only=True, save_weights_only=True, mode='max'), TensorBoard(log_dir=output_path), CSVLogger(filename=csv_path, separator=',', append=True), # LearningRateScheduler(unet_lr_scheduler) ] elif model_name == 'modify_unet': model_path = output_path + 'Modify_Unet_best_model.hdf5' csv_path = output_path + 'Modify_Unet.csv' img_tuple = (512, 512, 3) # batch_size = 3 model = get_modify_unet(input_shape=img_tuple, num_classes=nb_classes) if args.noweight: model.compile(optimizer=utils.SGD( lr=1e-02, decay=1e-6, momentum=0.9, nesterov=True, accumulator=np.ceil(float(15) / float(batch_size))), loss=weighted_loss, metrics=['binary_accuracy', dice_loss]) else: model.compile(optimizer=utils.Adam( lr=1e-02, accumulator=np.ceil(float(15) / float(batch_size))), loss=weighted_loss, metrics=['binary_accuracy', dice_loss]) callbacks = [ EarlyStopping(monitor='val_dice_loss', patience=8, verbose=1, min_delta=1e-05, mode='max'), ReduceLROnPlateau(monitor='val_dice_loss', factor=0.333, patience=3, verbose=1, epsilon=1e-04, mode='max', min_lr=3e-04), ModelCheckpoint(monitor='val_dice_loss', filepath=model_path, save_best_only=True, save_weights_only=True, mode='max'), TensorBoard(log_dir=output_path), CSVLogger(filename=csv_path, separator=',', append=True), # LearningRateScheduler(enet_lr_scheduler) ] elif model_name == 'modify_unet_V2': model_path = output_path + 'Modify_Unet_best_model.hdf5' csv_path = output_path + 'Modify_Unet.csv' img_tuple = (1024, 1024, 3) # batch_size = 5 model = get_modify_unet_V2(input_shape=img_tuple, num_classes=nb_classes) if args.noweight: model.compile(optimizer=utils.SGD( lr=1e-02, decay=1e-6, momentum=0.9, nesterov=True, accumulator=np.ceil(float(40) / float(batch_size))), loss=weighted_loss, metrics=['binary_accuracy', dice_loss]) else: model.compile(optimizer=utils.SGD( lr=1e-02, decay=1e-6, momentum=0.9, nesterov=True, accumulator=np.ceil(float(40) / float(batch_size))), loss=weighted_loss, metrics=['binary_accuracy', dice_loss]) callbacks = [ EarlyStopping(monitor='val_dice_loss', patience=8, verbose=1, min_delta=1e-05, mode='max'), ReduceLROnPlateau(monitor='val_dice_loss', factor=0.333, patience=3, verbose=1, epsilon=1e-04, mode='max', min_lr=1e-04), ModelCheckpoint(monitor='val_dice_loss', filepath=model_path, save_best_only=True, save_weights_only=True, mode='max'), TensorBoard(log_dir=output_path), CSVLogger(filename=csv_path, separator=',', append=True), # LearningRateScheduler(unet_lr_scheduler) ] # elif model_name == 'segnet': # model_path = output_path + 'SegNet_{val_loss:.4f}-{val_binary_accuracy:.4f}-{val_dice_loss:.5f}.hdf5' # csv_path = output_path + 'SegNet.csv' # image_size = 1024 # batch_size = 1 # model = SegNet(input_shape=(image_size, image_size, 3), classes=nb_classes) # model.compile(optimizer=optimizers.Adam(lr=0.01), loss=bce_dice_loss, metrics=['binary_accuracy', # dice_loss]) # # elif model_name == 'segnet_full': # model_path = output_path + 'SegNet_FULL_{val_loss:.4f}-{val_binary_accuracy:.4f}-{val_dice_loss:.5f}.hdf5' # csv_path = output_path + 'SegNet_FULL.csv' # image_size = 1024 # batch_size = 2 # model = SegNet(input_shape=(image_size, image_size, 3), classes=nb_classes) # model.compile(optimizer=optimizers.Adam(lr=0.01), loss=bce_dice_loss, metrics=['binary_accuracy', # dice_loss]) # elif model_name == 'enet_unpooling': # model_path = output_path + 'Enet_unpooling_{val_loss:.4f}-{val_binary_accuracy:.4f}.hdf5' # csv_path = output_path + 'Enet_unpooling.csv' # image_size = int(256) # model = get_enet_unpooling(input_shape=(image_size, image_size, 3), classes=nb_classes) # model.compile(optimizer=optimizers.Adam(lr=0.01), loss=losses.binary_crossentropy, metrics=['binary_accuracy', # dice_loss]) elif model_name == 'enet_naive_upsampling': model_path = output_path + 'Enet_naive_upsampling_best_model.hdf5' csv_path = output_path + 'Enet_naive_upsampling.csv' img_tuple = (512, 512, 3) # batch_size = 6 model = get_enet_naive_upsampling(input_shape=img_tuple, classes=nb_classes) if args.noweight: model.compile(optimizer=utils.SGD( lr=1e-02, decay=1e-6, momentum=0.9, nesterov=True, accumulator=np.ceil(float(14) / float(batch_size))), loss=weighted_loss, metrics=['binary_accuracy', dice_loss]) else: model.compile(optimizer=optimizers.Adam(lr=1e-02), loss=weighted_loss, metrics=['binary_accuracy', dice_loss]) callbacks = [ EarlyStopping(monitor='val_dice_loss', patience=8, verbose=1, min_delta=1e-05, mode='max'), ReduceLROnPlateau(monitor='val_dice_loss', factor=0.333, patience=3, verbose=1, epsilon=1e-05, mode='max', min_lr=3e-04), ModelCheckpoint(monitor='val_dice_loss', filepath=model_path, save_best_only=True, save_weights_only=True, mode='max'), TensorBoard(log_dir=output_path), CSVLogger(filename=csv_path, separator=',', append=True), # LearningRateScheduler(enet_lr_scheduler) ] # elif model_name == 'densenet_fc': # model_path = output_path + 'DenseNet_FC_{val_loss:.4f}-{val_binary_accuracy:.4f}-{val_dice_loss:.5f}.hdf5' # csv_path = output_path + 'DenseNet_FC.csv' # image_size = 256 # batch_size = 8 # base_model = DenseNetFCN(input_shape=(image_size, image_size, 3), classes=nb_classes, activation='sigmoid', include_top=False, batchsize=batch_size) # classify = Conv2D(nb_classes, (1, 1), activation='sigmoid')(base_model.output) # model = Model(inputs=base_model.input, outputs=classify) # model.compile(optimizer=optimizers.Adam(lr=0.01), loss=bce_dice_loss, metrics=['binary_accuracy', # dice_loss]) elif model_name == 'linknet': model_path = output_path + 'LinkNet_best_model.hdf5' csv_path = output_path + 'LinkNet.csv' img_tuple = (1024, 1024, 3) batch_size = 6 base_model = LinkNet(input_shape=img_tuple, classes=nb_classes) classify = Conv2D(nb_classes, (1, 1), activation='sigmoid')(base_model.output) model = Model(inputs=base_model.input, outputs=classify) model.compile(optimizer=utils.SGD(lr=1e-02, decay=1e-6, momentum=0.9, nesterov=True, accumulator=4), loss=weighted_loss, metrics=['binary_accuracy', dice_loss]) callbacks = [ EarlyStopping(monitor='val_dice_loss', patience=8, verbose=1, min_delta=1e-05, mode='max'), ReduceLROnPlateau(monitor='val_dice_loss', factor=0.333, patience=3, verbose=1, epsilon=1e-05, mode='max', min_lr=1e-04), ModelCheckpoint(monitor='val_dice_loss', filepath=model_path, save_best_only=True, save_weights_only=True, mode='max'), TensorBoard(log_dir=output_path), CSVLogger(filename=csv_path, separator=',', append=True), # LearningRateScheduler(linknet_lr_scheduler) ] elif model_name == 'linknet_res34': model_path = output_path + 'LinkNet_Res34_best_model.hdf5' csv_path = output_path + 'LinkNet_Res34.csv' img_tuple = (1024, 1024, 3) # batch_size = 12 base_model = LinkNet_Res34(input_shape=img_tuple, classes=nb_classes) classify = Conv2D(nb_classes, (1, 1), activation='sigmoid')(base_model.output) model = Model(inputs=base_model.input, outputs=classify) model.compile(optimizer=utils.SGD(lr=1e-02, decay=1e-6, momentum=0.9, nesterov=True, accumulator=np.ceil( float(84) / float(batch_size))), loss=weighted_loss_256, metrics=['binary_accuracy', dice_loss]) callbacks = [ EarlyStopping(monitor='val_dice_loss', patience=8, verbose=1, min_delta=1e-05, mode='max'), ReduceLROnPlateau(monitor='val_dice_loss', factor=0.333, patience=3, verbose=1, epsilon=1e-05, mode='max', min_lr=1e-04), ModelCheckpoint(monitor='val_dice_loss', filepath=model_path, save_best_only=True, save_weights_only=True, mode='max'), TensorBoard(log_dir=output_path), CSVLogger(filename=csv_path, separator=',', append=True), # LearningRateScheduler(unet_lr_scheduler) ] elif model_name == 'linknet_res50': model_path = output_path + 'LinkNet_Res50_{val_loss:.4f}-{val_binary_accuracy:.4f}-{val_dice_loss:.5f}.hdf5' csv_path = output_path + 'LinkNet_Res50.csv' img_tuple = (1280, 1280, 3) batch_size = 2 base_model = LinkNet_Res50(input_shape=img_tuple, classes=nb_classes) classify = Conv2D(nb_classes, (1, 1), activation='sigmoid')(base_model.output) model = Model(inputs=base_model.input, outputs=classify) model.compile(optimizer=utils.SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True, accumulator=10), loss=weighted_loss, metrics=['binary_accuracy', dice_loss]) callbacks = [ # EarlyStopping(monitor='val_dice_loss', # patience=8, # verbose=1, # min_delta=1e-05, # mode='max'), # ReduceLROnPlateau(monitor='val_dice_loss', # factor=0.333, # patience=3, # verbose=1, # epsilon=1e-05, # mode='max', # min_lr=1e-04), ModelCheckpoint(monitor='val_dice_loss', filepath=model_path, save_best_only=True, save_weights_only=True, mode='max'), TensorBoard(log_dir=output_path), CSVLogger(filename=csv_path, separator=',', append=True), LearningRateScheduler(unet_lr_scheduler) ] if model: train_batch_gen = iter( Batch_Generator(input_path=input_path, idx=ids_train_split, batch_size=batch_size, img_tuple=img_tuple, aug=True, sat=args.sat, hue=args.hue, val=args.val, rotate=args.rotate, gray=args.gray, contrast=args.contrast, brightness=args.brightness, shear=args.shear, shift=args.shift, scale=args.scale, lowq=args.lowq)) valid_batch_gen = iter( Batch_Generator(input_path=input_path, idx=ids_valid_split, batch_size=batch_size, img_tuple=img_tuple, aug=False)) print( 'Start to train model hue %r, sat %r, val %r, rotate %r, gray %r, contrast %r, brightness %r, shear %r, shift %r, scale %r, lowq %r, none weight %r' % ( args.hue, args.sat, args.val, args.rotate, args.gray, args.contrast, args.brightness, args.shear, args.shift, args.scale, args.lowq, args.noweight, )) model.fit_generator( generator=train_batch_gen, steps_per_epoch=np.ceil( float(len(ids_train_split)) / float(batch_size)), epochs=epochs, callbacks=callbacks, validation_data=valid_batch_gen, validation_steps=np.ceil( float(len(ids_valid_split)) / float(batch_size)), workers=6)
h1 = relu(nd.dot(X, W1) + b1)# 隐含层输出 非线性激活 output = nd.dot(h1, W2) + b2 return output ##Softmax和交叉熵损失函数 ## softmax 回归实现 exp(Xi)/(sum(exp(Xi))) 归一化概率 使得 10类概率之和为1 #交叉熵损失函数 softmax_cross_entropy = gluon.loss.SoftmaxCrossEntropyLoss() ## 开始训练 learning_rate = .5#学习率 epochs = 7 ##训练迭代训练集 次数 for epoch in range(epochs):##每迭代一次训练集 train_loss = 0.##损失 train_acc = 0. ##准确度 for data, label in train_data:#训练集 with autograd.record():#自动微分 output = net(data)#模型输出 向前传播 loss = softmax_cross_entropy(output, label)#计算损失 loss.backward()#向后传播 utils.SGD(params, learning_rate/batch_size)#随机梯度下降 训练更新参数 学习率递减 train_loss += nd.mean(loss).asscalar()#损失 train_acc += utils.accuracy(output, label)#准确度 test_acc = utils.evaluate_accuracy(test_data, net)#测试集测试 print("E次数 %d. 损失: %f, 训练准确度 %f, 测试准确度%f" % ( epoch, train_loss/len(train_data), train_acc/len(train_data), test_acc))
def main_worker(gpu, ngpus_per_node, args): global best_acc1 args.gpu = gpu if args.gpu is not None: print("Use GPU: {} for training".format(args.gpu)) if args.distributed: if args.dist_url == "env://" and args.rank == -1: args.rank = int(os.environ["RANK"]) if args.multiprocessing_distributed: # For multiprocessing distributed training, rank needs to be the # global rank among all the processes args.rank = args.rank * ngpus_per_node + gpu dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url, world_size=args.world_size, rank=args.rank) # create model if args.pretrained: print("=> using pre-trained model '{}'".format(args.arch)) # model = models.__dict__[args.arch](pretrained=True) model = resnet.__dict__[args.arch](pretrained=True) else: print("=> creating model '{}'".format(args.arch)) # model = models.__dict__[args.arch]() model = resnet.__dict__[args.arch]() if args.distributed: # For multiprocessing distributed, DistributedDataParallel constructor # should always set the single device scope, otherwise, # DistributedDataParallel will use all available devices. if args.gpu is not None: torch.cuda.set_device(args.gpu) model.cuda(args.gpu) # When using a single GPU per process and per # DistributedDataParallel, we need to divide the batch size # ourselves based on the total number of GPUs we have args.batch_size = int(args.batch_size / ngpus_per_node) args.workers = int( (args.workers + ngpus_per_node - 1) / ngpus_per_node) model = torch.nn.parallel.DistributedDataParallel( model, device_ids=[args.gpu]) else: model.cuda() # DistributedDataParallel will divide and allocate batch_size to all # available GPUs if device_ids are not set model = torch.nn.parallel.DistributedDataParallel(model) elif args.gpu is not None: torch.cuda.set_device(args.gpu) model = model.cuda(args.gpu) else: # DataParallel will divide and allocate batch_size to all available GPUs if args.arch.startswith('alexnet') or args.arch.startswith('vgg'): model.features = torch.nn.DataParallel(model.features) model.cuda() else: model = torch.nn.DataParallel(model).cuda() # define loss function (criterion) and optimizer criterion = nn.CrossEntropyLoss().cuda(args.gpu) optimizer = utils.SGD(model.parameters(), args.lr, momentum=args.momentum, weight_decay=args.weight_decay) # optionally resume from a checkpoint if args.resume: if os.path.isfile(args.resume): print("=> loading checkpoint '{}'".format(args.resume)) checkpoint = torch.load(args.resume) args.start_epoch = checkpoint['epoch'] best_acc1 = checkpoint['best_acc1'] if args.gpu is not None: # best_acc1 may be from a checkpoint from a different GPU best_acc1 = best_acc1.to(args.gpu) model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) print("=> loaded checkpoint '{}' (epoch {})".format( args.resume, checkpoint['epoch'])) else: print("=> no checkpoint found at '{}'".format(args.resume)) cudnn.benchmark = True # Data loading code traindir = os.path.join(args.data, 'train') valdir = os.path.join(args.data, 'val') normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) train_dataset = datasets.ImageFolder( traindir, transforms.Compose([ transforms.RandomResizedCrop(224), transforms.RandomHorizontalFlip(), transforms.ToTensor(), normalize, ])) if args.distributed: train_sampler = torch.utils.data.distributed.DistributedSampler( train_dataset) else: train_sampler = None train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=args.batch_size, shuffle=(train_sampler is None), num_workers=args.workers, pin_memory=True, sampler=train_sampler) val_loader = torch.utils.data.DataLoader(datasets.ImageFolder( valdir, transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), normalize, ])), batch_size=args.batch_size, shuffle=False, num_workers=args.workers, pin_memory=True) if args.evaluate: validate(val_loader, model, criterion, args) return #*quantize weights before training: for name, param in model.named_parameters(): print(name) if name.find('conv') > -1: param.data = qu(param.data) print("Weights quantized...") for epoch in range(args.start_epoch, args.epochs): if args.distributed: train_sampler.set_epoch(epoch) adjust_learning_rate(optimizer, epoch, args) # train for one epoch train(train_loader, model, criterion, optimizer, epoch, args) # evaluate on validation set acc1 = validate(val_loader, model, criterion, args) # remember best acc@1 and save checkpoint is_best = acc1 > best_acc1 best_acc1 = max(acc1, best_acc1) if not args.multiprocessing_distributed or ( args.multiprocessing_distributed and args.rank % ngpus_per_node == 0): save_checkpoint( { 'epoch': epoch + 1, 'arch': args.arch, 'state_dict': model.state_dict(), 'best_acc1': best_acc1, 'optimizer': optimizer.state_dict(), }, is_best)
train_data = copy.deepcopy(raw_data) dup_rawData = copy.deepcopy(train_data) ground_truth = train_data[:, 57] #(4001, 1) train_data = np.delete(train_data, -1, 1) print("dim of raw_data line 23", np.shape(train_data)) ################# training process, SGD ################# w = [] b = [] epochs = 300 learning_rate = 1e-5 # best so far (100, 1e-5), (300, 1000) w, b = utils.SGD(train_data, ground_truth, learning_rate, epochs) ################# training process, mini-batch SG ################# ''' w = [] b = [] epochs = 10000 learning_rate = 1e-7 # best so far (4000, 1e-7),(1000, 1e-7), w,b = utils.batchGD(train_data, ground_truth, 1000, learning_rate, epochs) ''' # open test data test_data = []