def test(self): tf.reset_default_graph() X = tf.placeholder("float", [None, None, None, cf.Channel]) Y = tf.placeholder("float", [None, cf.Class_num]) keep_prob = tf.placeholder("float") ## Load network model logits = Model(x=X, phase='Test') ## Secure GPU Memory config = tf.ConfigProto() config.gpu_options.allow_growth = True print('Test start !!') table_gt_pred = np.zeros((cf.Class_num, cf.Class_num), dtype=np.int) with tf.Session(config=config) as sess: saver = tf.train.Saver() saver.restore(sess, cf.Save_path) img_paths = get_imagelist() for img_path in img_paths: img, img_info = load_image(img_path) img = np.expand_dims(img, axis=0) gt = get_gt(img_path) pred = logits.eval(feed_dict={X: img, keep_prob: 1.0})[0] pred_label = np.argmax(pred) table_gt_pred[gt, pred_label] += 1 print(img_path, pred) for cls_ind in range(cf.Class_num): print(cf.Class_label[cls_ind], np.round(table_gt_pred[cls_ind], 3))
def main(args): # Image preprocessing transform = transforms.Compose([ transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)) ]) # Load vocabulary wrapper. with open(args['vocab_path'], 'rb') as f: vocab = pickle.load(f) # Build data loader data_loader = get_loader(args['image_dir'], args['caption_path'], vocab, transform, args['batch_size'], shuffle=True, num_workers=args['num_workers']) # Train the model model = Image2Seq((64, 64), vocab.word2idx) model.sess.run(tf.global_variables_initializer()) print("Model Compiled") for epoch in range(args['n_epoch']): for i, (images, captions, lengths) in enumerate(data_loader): loss = model.partial_fit(images.numpy(), captions.numpy(), lengths) print('[%d / %d] Loss: %.4f' % (i, len(data_loader), loss)) if i % 20 == 0: sample_image = load_image(args['sample_img']).numpy() model.infer(sample_image, vocab.idx2word)
def make_checkpoint_image(cfg, net, save_path): """ Function to use a test content image, run it through the network, and save it to track training progress. Args: --cfg: <dict> The training config dictionary Returns: None. Just writes the progress image. """ # Get image path for test image im_path = cfg['test_image'] # Load and preprocess image im = load_image(im_path, cfg) # Turn it into a batch im = im.unsqueeze(0) # Pass it through the network styled = net(im)[0] # Post process image styled = postprocess_image(styled.detach()) # Convert to BGR and write output styled = cv2.cvtColor(styled, cv2.COLOR_RGB2BGR) cv2.imwrite(save_path, styled) return
def get_batch(step, questions, answers, images_paths, answers_vocab_len): batch_start = (step * batch_size) % len(questions) batch_in_questions = questions[batch_start:batch_start + batch_size] batch_in_images = list() batch_out = np.zeros((batch_size, answers_vocab_len)) for i in range(batch_start, batch_start + len(batch_in_questions)): batch_in_images.append(load_image(images_paths[i])) batch_out[i - batch_start, answers[i] - 1] = 1 tmp = batch_size - len(batch_in_questions) if tmp > 0: for i in range(0, tmp): batch_out[i + len(batch_in_questions), answers[i] - 1] = 1 batch_in_images.append(load_image(images_paths[i])) batch_in_questions = np.concatenate( (batch_in_questions, questions[0:tmp]), axis=0) return batch_in_questions, np.asarray(batch_in_images), batch_out
def get_batch_for_test(step, questions, answers, images_paths, answers_vocab_len): batch_start = (step * batch_size) % len(questions) batch_in_questions = questions[batch_start:batch_start + batch_size] batch_in_images = list() batch_out = np.zeros((len(batch_in_questions), answers_vocab_len)) for i in range(batch_start, batch_start + len(batch_in_questions)): batch_in_images.append(load_image(images_paths[i])) batch_out[i - batch_start, answers[i] - 1] = 1 return batch_in_questions, np.asarray(batch_in_images), batch_out, len( batch_in_questions)
def train(configuration): """ the main training loop :param configuration: the config file :return: """ image_path = configuration['image_path'] print('using images from {}'.format(image_path)) image_saving_path_mean = configuration['image_saving_path_mean'] print('saving result images with mean loss to {}'.format(image_saving_path_mean)) image_saving_path_mean_std = configuration['image_saving_path_mean_std'] print('saving result images with mean + std loss to {}'.format(image_saving_path_mean_std)) image_saving_path_mean_std_skew = configuration['image_saving_path_mean_std_skew'] print('saving result images with mean + std + skew loss to {}'.format(image_saving_path_mean_std_skew)) image_saving_path_mean_std_skew_kurtosis = configuration['image_saving_path_mean_std_skew_kurtosis'] print('saving result images with mean + std + skew + kurtosis loss to {}'.format(image_saving_path_mean_std_skew_kurtosis)) model_dir = configuration['model_dir'] print('vgg-19 model dir is {}'.format(model_dir)) vgg_model = get_vgg_model(configuration) print('got vgg model') number_style_images, style_image_file_paths = get_images(configuration) print('got {} style images'.format(number_style_images)) all_images = [] for i in range(1, 6): print('using style loss module: {}'.format(i)) style_loss_module = get_loss_module(i) for j in range(number_style_images): print('computing image {} with style loss module {}'.format(j, i)) style_image = load_image(style_image_file_paths[j]) layer_images = [style_image.squeeze(0)] model, style_losses = get_full_style_model(configuration, vgg_model, style_image, style_loss_module) for k in range(4): print('computing loss at layer {}, image {}, loss module {}'.format(k, j, i)) torch.manual_seed(13) image_noise = torch.randn(style_image.data.size()).to(device) layer_images += [train_full_style_model(model, style_losses, image_noise, k, i * (k+1) * 20000) .squeeze(0)] print('saving images at the different layers') save_layer_images(configuration, layer_images, i, j) all_images += [layer_images] print('saving the side-by-side comparisons') save_all_images(configuration, all_images, number_style_images)
def train_mmd(configuration): """ training loop utilizing the MMD loss :param configuration: the config file :return: """ image_path = configuration['image_path'] print('using images from {}'.format(image_path)) image_saving_path_mmd = configuration['image_saving_path_mmd'] print('saving result images with mmd loss to {}'.format(image_saving_path_mmd)) model_dir = configuration['model_dir'] print('vgg-19 model dir is {}'.format(model_dir)) vgg_model = get_vgg_model(configuration) print('got vgg model') number_style_images, style_image_file_paths = get_images(configuration) print('got {} style images'.format(number_style_images)) i = 6 print('using mmd loss module: {}'.format(i)) style_loss_module = get_loss_module(i) for j in range(number_style_images): print('computing image {} with style loss module {}'.format(j, i)) style_image = load_image(style_image_file_paths[j]) layer_images = [style_image.squeeze(0)] model, style_losses = get_full_style_model(configuration, vgg_model, style_image, style_loss_module) for k in range(4): print('computing loss at layer {}, image {}, loss module {}'.format(k, j, i)) torch.manual_seed(13) image_noise = torch.randn(style_image.data.size()).to(device) steps = (k+2) * 50000 style_weight = 1000 layer_images += [train_full_style_model(model, style_losses, image_noise, k, steps, style_weight, early_stopping=True).squeeze(0)] # for debug save_layer_images(configuration, layer_images, i, j) print('saving images at the different layers') save_layer_images(configuration, layer_images, i, j) print('finished')
def read_tf_weight(self): model = adapted_deeplab_tf.DeepLab( num_classes=self.nSketchClasses, upsample_mode=self.upsample_mode, ignore_class_bg=self.ignore_class_bg) tfconfig = tf.ConfigProto() tfconfig.gpu_options.allow_growth = True sess = tf.Session(config=tfconfig) sess.run(tf.global_variables_initializer()) ckpt = tf.train.get_checkpoint_state(self.tf_model_dir) snapshot_loader = tf.train.Saver() print('Trained model found, loaded', ckpt.model_checkpoint_path) snapshot_loader.restore(sess, ckpt.model_checkpoint_path) weight_keys = [var.name[:-2] for var in tf.global_variables()] weight_vals = sess.run(tf.global_variables()) for weight_key, weight_val in zip(weight_keys, weight_vals): if 'factor' not in weight_key and 'Adam' not in weight_key and 'beta1_power' not in weight_key \ and 'beta2_power' not in weight_key and 'global_step' not in weight_key: # print(weight_key, np.array(weight_val).shape) self.tf_weight_dict[weight_key] = weight_val print('Obtain tf_weight_dict done!') # get output if self.display: infer_image, infer_image_raw = load_image( self.image_path, self.config.mean, True) # shape = [1, H, W, 3] feed_dict = {model.images: infer_image, model.labels: 0} self.pred_label_tf = sess.run( [model.pred_label], feed_dict=feed_dict)[0] # [1, H, W, 1] if self.ignore_class_bg: pred_label_tf = self.pred_label_tf + 1 else: pred_label_tf = self.pred_label_tf pred_label_tf = np.squeeze(pred_label_tf) pred_label_tf[infer_image_raw[:, :, 0] != 0] = 0 # [H, W] visualize_semantic_segmentation(pred_label_tf, self.colorMap)
def train_gram(configuration): """ training loop utilizing the Gram matrix loss :param configuration: the config file :return: """ image_path = configuration['image_path'] print('using images from {}'.format(image_path)) image_saving_path_gram = configuration['image_saving_path_gram'] print('saving result images with gram loss to {}'.format(image_saving_path_gram)) model_dir = configuration['model_dir'] print('vgg-19 model dir is {}'.format(model_dir)) vgg_model = get_vgg_model(configuration) print('got vgg model') number_style_images, style_image_file_paths = get_images(configuration) print('got {} style images'.format(number_style_images)) all_images = [] i = 5 print('using gram loss module: {}'.format(i)) style_loss_module = get_loss_module(i) for j in range(number_style_images): print('computing image {} with style loss module {}'.format(j, i)) style_image = load_image(style_image_file_paths[j]) layer_images = [style_image.squeeze(0)] model, style_losses = get_full_style_model(configuration, vgg_model, style_image, style_loss_module) for k in range(4): print('computing loss at layer {}, image {}, loss module {}'.format(k, j, i)) torch.manual_seed(13) image_noise = torch.randn(style_image.data.size()).to(device) steps = i * (k+1) * 20000 layer_images += [train_full_style_model(model, style_losses, image_noise, k, steps).squeeze(0)] print('saving images at the different layers') save_layer_images(configuration, layer_images, i, j) all_images += [layer_images] print('finished')
def run(cfg): """Runs the neural style transfer""" # Load image and make batch image = load_image(cfg['content_image'], rescale=False) image = image.unsqueeze(0) # Set device if gpu is available device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') # Print alert if device is cpu if device == 'cpu': print('No GPU detected. This may take longer to run...') # Figure out if artist is VanGoh (VanGoh weights have slightly different shape) vangoh = False if cfg['artist'] == 'VanGoh': vangoh = True # Build network net = ImageTransformationNet(vangoh=vangoh).to(device) # Load trained weights artist = torch.load(f'./trained_weights/{cfg["artist"]}.pth') net.load_state_dict(artist['net_state_dict']) # net.load_state_dict(artist) # Put image through net processed_im = net(image)[0] # Postprocess image output = postprocess_image(processed_im) # Write output to output file output = cv2.cvtColor(output, cv2.COLOR_RGB2BGR) cv2.imwrite(cfg['output_file'], output) print('Output image written successfully!') return
def train(cfg): # Set device if gpu is available device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') # Build network net = ImageTransformationNet().to(device) # Setup optimizer optimizer = optim.Adam(net.parameters()) # Load state if resuming training if cfg['resume']: checkpoint = torch.load(cfg['resume']) net.load_state_dict(checkpoint['net_state_dict']) optimizer.load_state_dict(checkpoint['opt_state_dict']) # Get starting epoch and batch (expects weight file in form EPOCH_<>_BATCH_<>.pt) parts = cfg['resume'].split('_') first_epoch = int(checkpoint['epoch']) first_batch = int(parts[-1].split('.')[0]) # Setup dataloader train_data = tqdm(build_data_loader(cfg), initial=first_batch) else: # Setup dataloader train_data = tqdm(build_data_loader(cfg)) # Set first epoch and batch first_epoch = 1 first_batch = 0 # Fetch style image and style grams style_im = load_image(cfg['style_image'], cfg) style_grams = get_style_grams(style_im, cfg) # Setup log file if specified log_dir = Path('logs') log_dir.mkdir(parents=True, exist_ok=True) if cfg['log_file'] and not cfg['resume']: today = datetime.datetime.today().strftime('%m/%d/%Y') header = f'Feed-Forward Style Transfer Training Log - {today}' with open(cfg['log_file'], 'w+') as file: file.write(header + '\n\n') # Setup log CSV if specified if cfg['csv_log_file'] and not cfg['resume']: utils.setup_csv(cfg) for epoch in range(first_epoch, cfg['epochs'] + 1): # Keep track of per epoch loss content_loss = 0 style_loss = 0 total_var_loss = 0 train_loss = 0 num_batches = 0 # Setup first batch to start enumerate at proper place if epoch == first_epoch: start = first_batch else: start = 0 for i, batch in enumerate(train_data, start=start): batch = batch.to(device) # Put batch through network batch_styled = net(batch) # Get vgg activations for styled and unstyled batch features = vgg_activations(batch_styled) content_features = vgg_activations(batch) # Get loss c_loss, s_loss = perceptual_loss(features=features, content_features=content_features, style_grams=style_grams, cfg=cfg) tv_loss = total_variation_loss(batch_styled, cfg) total_loss = c_loss + s_loss + tv_loss # Backpropogate total_loss.backward() # Do one step of optimization optimizer.step() # Clear gradients before next batch optimizer.zero_grad() # Update summary statistics with torch.no_grad(): content_loss += c_loss.item() style_loss += s_loss.item() total_var_loss += tv_loss.item() train_loss += total_loss.item() num_batches += 1 # Update progress bar avg_loss = round(train_loss / num_batches, 2) avg_c_loss = round(content_loss / num_batches, 2) avg_s_loss = round(style_loss / num_batches, 1) avg_tv_loss = round(total_var_loss / num_batches, 3) train_data.set_description( f'C - {avg_c_loss} | S - {avg_s_loss} | TV - {avg_tv_loss} | Total - {avg_loss}' ) train_data.refresh() # Create progress image if specified if cfg['image_checkpoint'] and ((i + 1) % cfg['image_checkpoint'] == 0): save_path = str( Path( cfg['image_checkpoint_dir'], f'EPOCH_{str(epoch).zfill(3)}_BATCH_{str(i+1).zfill(5)}.png' )) utils.make_checkpoint_image(cfg, net, save_path) # Save weights if specified if cfg['save_checkpoint'] and ((i + 1) % cfg['save_checkpoint'] == 0): save_path = str( Path( cfg['save_checkpoint_dir'], f'EPOCH_{str(epoch).zfill(3)}_BATCH_{str(i+1).zfill(5)}.pth' )) checkpoint = { 'epoch': epoch, 'net_state_dict': net.state_dict(), 'opt_state_dict': optimizer.state_dict(), 'loss': avg_loss } torch.save(checkpoint, save_path) # Write progress row to CSV if cfg['csv_checkpoint'] and ((i + 1) % cfg['csv_checkpoint'] == 0): row = [ epoch, i + 1, avg_c_loss, avg_s_loss, avg_tv_loss, avg_loss ] utils.write_progress_row(cfg, row) # Write loss at end of each epoch if cfg['log_file']: avg_loss = round(train_loss / num_batches, 4) line = f'EPOCH {epoch} | Loss - {avg_loss}' with open(cfg['log_file'], 'a') as file: file.write(line + '\n') # Save network if specified if cfg['epoch_save_checkpoint'] and ( epoch % cfg['epoch_save_checkpoint'] == 0): save_path = str( Path(cfg['save_checkpoint_dir'], f'EPOCH_{str(epoch).zfill(3)}.pth')) checkpoint = { 'epoch': epoch, 'net_state_dict': net.state_dict(), 'opt_state_dict': optimizer.state_dict(), 'loss': round(train_loss / num_batches, 4) } torch.save(checkpoint, save_path)
from keras.utils import np_utils from keras.models import load_model import data_loader from keras.models import model_from_json im_size = 320 model = densenet.DenseNet(nb_classes=1, img_dim=(320, 320, 1), depth=22, nb_dense_block=4, growth_rate=12, nb_filter=16, dropout_rate=0.2, weight_decay=1E-4) model.load_weights('./save_models/[email protected]') X_valid_path, Y_valid = data_loader.load_path(root_path='./valid/XR_HUMERUS', size=im_size) X_valid = data_loader.load_image(X_valid_path, im_size) y1 = model.predict(X_valid, batch_size=None, verbose=0, steps=None) j = len(y1) for i in range(0, j): if y1[i] > 0.5: print(X_valid_path[i], ":\t", "Positive\t", y1[i]) else: print(X_valid_path[i], ":\t", "Negative\t", y1[i])
__author__ = 'Mohammad' import tensorflow as tf import numpy as np from data_loader import load_image sess = tf.Session() saver = tf.train.import_meta_graph( 'data/tensorflow-resnet-pretrained-20160509/ResNet-L152.meta') saver.restore(sess, 'data/tensorflow-resnet-pretrained-20160509/ResNet-L152.ckpt') graph = tf.get_default_graph() images = graph.get_tensor_by_name("images:0") img = load_image("data/train2014/COCO_train2014_000000000009.jpg") img_features = graph.get_tensor_by_name("avg_pool:0") features = sess.run(img_features, {images: img[np.newaxis, :]}) print features[0], features[0].shape
def run_MURA(batch_size, nb_epoch, depth, nb_dense_block, nb_filter, growth_rate, dropout_rate, learning_rate, weight_decay, plot_architecture): """ Run MURA experiments :param batch_size: int -- batch size :param nb_epoch: int -- number of training epochs :param depth: int -- network depth :param nb_dense_block: int -- number of dense blocks :param nb_filter: int -- initial number of conv filter :param growth_rate: int -- number of new filters added by conv layers :param dropout_rate: float -- dropout rate :param learning_rate: float -- learning rate :param weight_decay: float -- weight decay :param plot_architecture: bool -- whether to plot network architecture """ ################### # Data processing # ################### #/home/yu/Documents/tensorflow/MURA/MURA-v1.1 # the path of MURA dataset im_size = 320 #测试修改参数 size root_path nb_epoch nb_dense_block X_train_path, Y_train = data_loader.load_path(root_path='../train', size=im_size) X_valid_path, Y_valid = data_loader.load_path(root_path='../valid', size=im_size) X_valid = data_loader.load_image(X_valid_path, im_size) #提前加载验证集 Y_valid = np.asarray(Y_valid) nb_classes = 1 img_dim = (im_size, im_size, 1) #加上最后一个维度,类型为tuple ################### # Construct model # ################### model = densenet.DenseNet(nb_classes, img_dim, depth, nb_dense_block, growth_rate, nb_filter, dropout_rate=dropout_rate, weight_decay=weight_decay) # Model output model.summary() # Build optimizer opt = Adam(lr=learning_rate, beta_1=0.9, beta_2=0.999, epsilon=1e-08) model.compile(loss='binary_crossentropy', optimizer=opt, metrics=["accuracy"]) if plot_architecture: from keras.utils import plot_model plot_model(model, to_file='./figures/densenet_archi.png', show_shapes=True) #################### # Network training # #################### print("Start Training") list_train_loss = [] list_valid_loss = [] list_learning_rate = [] best_record = [100, 0, 100, 100] #记录最优 [验证集损失函数值,准确率,训练集数据集loss差值,acc差值] start_time = datetime.datetime.now() for e in range(nb_epoch): if e == int(0.25 * nb_epoch): K.set_value(model.optimizer.lr, np.float32(learning_rate / 10.)) if e == int(0.5 * nb_epoch): K.set_value(model.optimizer.lr, np.float32(learning_rate / 50.)) if e == int(0.75 * nb_epoch): K.set_value(model.optimizer.lr, np.float32(learning_rate / 100.)) split_size = batch_size num_splits = len(X_train_path) / split_size arr_all = np.arange(len(X_train_path)).astype(int) random.shuffle(arr_all) #随机打乱index索引顺序 arr_splits = np.array_split(arr_all, num_splits) l_train_loss = [] batch_train_loss = [] start = datetime.datetime.now() for i, batch_idx in enumerate(arr_splits): X_batch_path, Y_batch = [], [] for idx in batch_idx: X_batch_path.append(X_train_path[idx]) Y_batch.append(Y_train[idx]) X_batch = data_loader.load_image(Path=X_batch_path, size=im_size) Y_batch = np.asarray(Y_batch) train_logloss, train_acc = model.train_on_batch(X_batch, Y_batch) l_train_loss.append([train_logloss, train_acc]) batch_train_loss.append([train_logloss, train_acc]) if i % 100 == 0: loss_1, acc_1 = np.mean(np.array(l_train_loss), 0) loss_2, acc_2 = np.mean(np.array(batch_train_loss), 0) batch_train_loss = [] #当前100batch的损失函数和准确率 print( '[Epoch {}/{}] [Batch {}/{}] [Time: {}] [all_batchs--> train_epoch_logloss: {:.5f}, train_epoch_acc:{:.5f}] ' .format(e + 1, nb_epoch, i, len(arr_splits), datetime.datetime.now() - start, loss_1, acc_1), '[this_100_batchs-->train_batchs_logloss: {:.5f}, train_batchs_acc:{:.5f}]' .format(loss_2, acc_2)) # 运行验证集 valid_logloss, valid_acc = model.evaluate(X_valid, Y_valid, verbose=0, batch_size=64) list_train_loss.append(np.mean(np.array(l_train_loss), 0).tolist()) list_valid_loss.append([valid_logloss, valid_acc]) list_learning_rate.append(float(K.get_value(model.optimizer.lr))) # to convert numpy array to json serializable print('[Epoch %s/%s] [Time: %s, Total_time: %s]' % (e + 1, nb_epoch, datetime.datetime.now() - start, datetime.datetime.now() - start_time), end='') print( '[train_loss_and_acc:{:.5f} {:.5f}] [valid_loss_acc:{:.5f} {:.5f}]' .format(list_train_loss[-1][0], list_train_loss[-1][1], list_valid_loss[-1][0], list_valid_loss[-1][1])) d_log = {} d_log["batch_size"] = batch_size d_log["nb_epoch"] = nb_epoch d_log["optimizer"] = opt.get_config() d_log["train_loss"] = list_train_loss d_log["valid_loss"] = list_valid_loss d_log["learning_rate"] = list_learning_rate json_file = os.path.join('./log/experiment_log_MURA.json') with open(json_file, 'w') as fp: json.dump(d_log, fp, indent=4, sort_keys=True) record = [ valid_logloss, valid_acc, abs(valid_logloss - list_train_loss[-1][0]), abs(valid_acc - list_train_loss[-1][1]), ] if ((record[0] <= best_record[0]) & (record[1] >= best_record[1])): if e <= int(0.25 * nb_epoch) | (record[2] <= best_record[2]) & ( record[3] <= best_record[3]): #四分之一epoch之后加入差值判定 best_record = record #记录最小的 [验证集损失函数值,准确率,训练集数据loss差值,acc差值] print('saving the best model:epoch', e + 1, best_record) model.save('save_models/best_MURA_modle@epochs{}.h5'.format(e + 1)) model.save('save_models/MURA_modle@epochs{}.h5'.format(e + 1))
X_train_path, Y_train = data_loader.load_path( root_path='/Users/curlyfu/Documents/MURA-6105/MURA-v1.1/train1', size=im_size) X_valid_path, Y_valid = data_loader.load_path( root_path='/Users/curlyfu/Documents/MURA-6105/MURA-v1.1/valid', size=im_size) from sklearn.model_selection import train_test_split # x is the feature, and y is the label. X_train_path, X_test_path, Y_train, Y_test = train_test_split(X_train_path, Y_train, test_size=0.2) print("loading train set......") X_train = data_loader.load_image(X_train_path, im_size) # load trainset print("loading train set finished") print("Y_train....") Y_train = np.asarray(Y_train) print("Y_train finished") print("loading valid set......") X_valid = data_loader.load_image(X_valid_path, im_size) # loadvalidset Y_valid = np.asarray(Y_valid) print("loading test set......") X_test = data_loader.load_image(X_test_path, im_size) # loadtest Y_test = np.asarray(Y_test) nb_classes = 1 img_dim = (im_size, im_size, 1) # plus the last dimension, type tuple
def run_MURA(batch_size, nb_epoch, depth, nb_dense_block, nb_filter, growth_rate, dropout_rate, learning_rate, weight_decay, plot_architecture): """ Run MURA experiments :parameter batch_size: int -- batch size :parameter nb_epoch: int -- number of training epochs :parameter depth: int -- network depth :parameter nb_dense_block: int -- number of dense blocks :parameter nb_filter: int -- initial number of conv filter :parameter growth_rate: int -- number of new filters added by conv layers :parameter dropout_rate: float -- dropout rate :parameter learning_rate: float -- learning rate :parameter weight_decay: float -- weight decay :parameter plot_architecture: bool -- whether to plot network architecture """ ################### # Data processing # ################### im_size = 320 #Test modification parameters size root_path nb_epoch nb_dense_block X_train_path, Y_train = data_loader.load_path( root_path='./train/XR_HUMERUS', size=im_size) X_valid_path, Y_valid = data_loader.load_path( root_path='./valid/XR_HUMERUS', size=im_size) X_valid = data_loader.load_image( X_valid_path, im_size) #Load verification set ahead of time Y_valid = np.asarray(Y_valid) nb_classes = 1 img_dim = (im_size, im_size, 1) #Plus the last dimension, type is tuple ################### # Construct model # ################### model = densenet.DenseNet(nb_classes, img_dim, depth, nb_dense_block, growth_rate, nb_filter, dropout_rate=dropout_rate, weight_decay=weight_decay) # Model output model.summary() # Build optimizer opt = Adam(lr=learning_rate, beta_1=0.9, beta_2=0.999, epsilon=1e-08) model.compile(loss='binary_crossentropy', optimizer=opt, metrics=["accuracy"]) if plot_architecture: from keras.utils import plot_model plot_model(model, to_file='./figures/densenet_archi.png', show_shapes=True) #################### # Network training # #################### print("Start Training") list_train_loss = [] list_valid_loss = [] list_learning_rate = [] best_record = [ 100, 0, 100, 100 ] #Recording optimal [verification set loss function value, accuracy rate, training set data set loss difference,acc difference] start_time = datetime.datetime.now() for e in range(nb_epoch): if e == int(0.25 * nb_epoch): K.set_value(model.optimizer.lr, np.float32(learning_rate / 10.)) if e == int(0.5 * nb_epoch): K.set_value(model.optimizer.lr, np.float32(learning_rate / 50.)) if e == int(0.75 * nb_epoch): K.set_value(model.optimizer.lr, np.float32(learning_rate / 100.)) split_size = batch_size num_splits = len(X_train_path) / split_size arr_all = np.arange(len(X_train_path)).astype(int) random.shuffle(arr_all) #Randomly disrupted index order arr_splits = np.array_split(arr_all, num_splits) l_train_loss = [] batch_train_loss = [] start = datetime.datetime.now() for i, batch_idx in enumerate(arr_splits): X_batch_path, Y_batch = [], [] for idx in batch_idx: X_batch_path.append(X_train_path[idx]) Y_batch.append(Y_train[idx]) X_batch = data_loader.load_image(Path=X_batch_path, size=im_size) Y_batch = np.asarray(Y_batch) train_logloss, train_acc = model.train_on_batch(X_batch, Y_batch) l_train_loss.append([train_logloss, train_acc]) batch_train_loss.append([train_logloss, train_acc]) if i % 100 == 0: loss_1, acc_1 = np.mean(np.array(l_train_loss), 0) loss_2, acc_2 = np.mean(np.array(batch_train_loss), 0) batch_train_loss = [ ] #Current 100 batch loss function and accuracy print( '[Epoch {}/{}] [Batch {}/{}] [Time: {}] [all_batchs--> train_epoch_logloss: {:.5f}, train_epoch_acc:{:.5f}] ' .format(e + 1, nb_epoch, i, len(arr_splits), datetime.datetime.now() - start, loss_1, acc_1), '[this_100_batchs-->train_batchs_logloss: {:.5f}, train_batchs_acc:{:.5f}]' .format(loss_2, acc_2)) # Run verification set valid_logloss, valid_acc = model.evaluate(X_valid, Y_valid, verbose=0, batch_size=64) list_train_loss.append(np.mean(np.array(l_train_loss), 0).tolist()) list_valid_loss.append([valid_logloss, valid_acc]) list_learning_rate.append(float(K.get_value(model.optimizer.lr))) # to convert numpy array to json serializable print('[Epoch %s/%s] [Time: %s, Total_time: %s]' % (e + 1, nb_epoch, datetime.datetime.now() - start, datetime.datetime.now() - start_time), end='') print( '[train_loss_and_acc:{:.5f} {:.5f}] [valid_loss_acc:{:.5f} {:.5f}]' .format(list_train_loss[-1][0], list_train_loss[-1][1], list_valid_loss[-1][0], list_valid_loss[-1][1])) d_log = {} d_log["batch_size"] = batch_size d_log["nb_epoch"] = nb_epoch d_log["optimizer"] = opt.get_config() d_log["train_loss"] = list_train_loss d_log["valid_loss"] = list_valid_loss d_log["learning_rate"] = list_learning_rate json_file = os.path.join('./log/experiment_log_MURA.json') with open(json_file, 'w') as fp: json.dump(d_log, fp, indent=4, sort_keys=True) record = [ valid_logloss, valid_acc, abs(valid_logloss - list_train_loss[-1][0]), abs(valid_acc - list_train_loss[-1][1]), ] if ((record[0] <= best_record[0]) & (record[1] >= best_record[1])): if e <= int(0.25 * nb_epoch) | (record[2] <= best_record[2]) & ( record[3] <= best_record[3] ): #Add a difference judgment after a quarter epoch best_record = record #Record the smallest [validation set loss function value, accuracy rate, training set data loss difference, acc difference] print('saving the best model:epoch', e + 1, best_record) model.save('save_models/best_MURA_modle@epochs{}.h5'.format(e + 1)) model.save('save_models/MURA_modle@epochs{}.h5'.format(e + 1))
def train(configuration): """ this is the main training loop :param configuration: the config :return: """ image_saving_path = configuration['image_saving_path'] print('saving result images to {}'.format(image_saving_path)) model_dir = configuration['model_dir'] print('vgg-19 model dir is {}'.format(model_dir)) vgg_model = get_vgg_model(configuration) print('got vgg model') style_image_path = configuration['style_image_path'] number_style_images, style_image_file_paths = get_images(style_image_path) print('got {} style images'.format(number_style_images)) print('using the style images from path: {}'.format(style_image_path)) content_image_path = configuration['content_image_path'] number_content_images, content_image_file_paths = get_images( content_image_path) print('got {} content images'.format(number_content_images)) print('using the content images from path: {}'.format(content_image_path)) steps = configuration['steps'] print('training for {} steps'.format(steps)) content_weight = configuration['content_weight'] style_weight = configuration['style_weight'] print('content weight: {}, style weight: {}'.format( content_weight, style_weight)) lr = configuration['lr'] print('using a learning rate of {}'.format(lr)) for i in range(number_style_images): print('style image {}'.format(i)) for j in range(number_content_images): images = [] print('content image {}'.format(j)) style_image = load_image(style_image_file_paths[i]) content_image = load_image(content_image_file_paths[j]) images += [style_image.squeeze(0).cpu()] print('got style image') images += [content_image.squeeze(0).cpu()] print('got content image') for k in range(1, 6): print('training transfer image with loss {}'.format(k)) loss_writer = LossWriter( os.path.join( configuration['folder_structure'].get_parent_folder(), './loss/loss')) loss_writer.write_header(columns=[ 'iteration', f'style_loss_{k}', f'content_loss_{k}', f'loss_{k}' ]) torch.manual_seed(1) image_noise = torch.randn(style_image.data.size()).to(device) model, style_losses, content_losses = get_full_style_model( configuration, vgg_model, style_image, content_image, get_style_loss_module(k), get_content_loss_module()) # this is to align the loss magnitudes of Gram matrix loss and moment loss if k == 1: style_weight *= 100 img = train_neural_style_transfer( model, lr, style_losses, content_losses, image_noise, steps, style_weight, content_weight, loss_writer).squeeze(0).cpu() images += [img.clone()] save_single_image(configuration, img, -k, -k) print('got transfer image') save_image(configuration, images, i, j)
def train_mmd(configuration): """ this is the MMD training loop :param configuration: the config :return: """ image_saving_path = configuration['image_saving_path'] print('saving result images to {}'.format(image_saving_path)) model_dir = configuration['model_dir'] print('vgg-19 model dir is {}'.format(model_dir)) vgg_model = get_vgg_model(configuration) print('got vgg model') style_image_path = configuration['style_image_path'] number_style_images, style_image_file_paths = get_images(style_image_path) print('got {} style images'.format(number_style_images)) print('using the style images from path: {}'.format(style_image_path)) content_image_path = configuration['content_image_path'] number_content_images, content_image_file_paths = get_images( content_image_path) print('got {} content images'.format(number_content_images)) print('using the content images from path: {}'.format(content_image_path)) loss_writer = LossWriter( os.path.join(configuration['folder_structure'].get_parent_folder(), './loss/loss')) loss_writer.write_header( columns=['iteration', 'style_loss', 'content_loss', 'loss']) print(style_image_file_paths) print(content_image_file_paths) images = [] for i in range(number_style_images): print('style image {}'.format(i)) for j in range(number_content_images): style_image = load_image(style_image_file_paths[i]) content_image = load_image(content_image_file_paths[j]) images += [style_image.squeeze(0).cpu()] print('got style image') images += [content_image.squeeze(0).cpu()] print('got content image') print('training transfer image with loss {} (MMD loss)'.format(2)) torch.manual_seed(1) image_noise = torch.randn(style_image.data.size()).to(device) model, style_losses, content_losses = get_full_style_model( configuration, vgg_model, style_image, content_image, get_style_loss_module(2), get_content_loss_module()) steps = configuration['steps'] print('training for {} steps'.format(steps)) content_weight = configuration['content_weight'] style_weight = configuration['style_weight'] print('content weight: {}, style weight: {}'.format( content_weight, style_weight)) lr = configuration['lr'] print('learning rate: {}'.format(lr)) img = train_neural_style_transfer(model, lr, style_losses, content_losses, image_noise, steps, style_weight, content_weight, loss_writer).squeeze(0).cpu() save_image(configuration, img, j, i) print('got transfer image')
config = tf.ConfigProto( device_count = {'GPU': 0} ) with tf.Session(config=config) as sess: init_op = tf.global_variables_initializer() model = sess.run(init_op) if os.path.isfile(os.getcwd() + "/" + cfg.weights_dir + "/checkpoint"): saver.restore(sess, model_file) print("Restored model") yolo.set_training(False) anchors = np.reshape(np.array(cfg.anchors), [-1, 2]) images = np.array([load_image(sys.argv[1])]) img = images[0] #normalise data between 0 and 1 imgs = np.array(images)/127.5-1 boxes = sess.run(yolo.output, feed_dict={ yolo.x: imgs, yolo.anchors: anchors, }) proc_boxes = yolo.convert_net_to_bb(boxes, filter_top=True).tolist()[0] raw_img = load_raw_image(sys.argv[1])
def segment_main(**kwargs): mode = kwargs['mode'] mu = FLAGS.mean if FLAGS.ignore_class_bg: nSketchClasses = FLAGS.nSketchClasses - 1 print('Ignore BG;', nSketchClasses, 'classes') else: nSketchClasses = FLAGS.nSketchClasses print('Not Ignore BG;', nSketchClasses, 'classes') data_aug = FLAGS.data_aug if mode == 'train' else False model = adapted_deeplab_model.DeepLab(num_classes=nSketchClasses, lrn_rate=FLAGS.learning_rate, lrn_rate_end=FLAGS.learning_rate_end, optimizer=FLAGS.optimizer, upsample_mode=FLAGS.upsample_mode, data_aug=data_aug, image_down_scaling=FLAGS.image_down_scaling, ignore_class_bg=FLAGS.ignore_class_bg, mode=mode) snapshot_saver = tf.train.Saver(max_to_keep=5) tfconfig = tf.ConfigProto() tfconfig.gpu_options.allow_growth = True sess = tf.Session(config=tfconfig) sess.run(tf.global_variables_initializer()) snapshot_dir = os.path.join(FLAGS.outputs_base_dir, FLAGS.snapshot_folder_name) os.makedirs(snapshot_dir, exist_ok=True) ckpt = tf.train.get_checkpoint_state(snapshot_dir) if not ckpt: if mode == 'train': pretrained_model = FLAGS.resnet_pretrained_model_path load_var = {var.op.name: var for var in tf.global_variables() if var.op.name.startswith('ResNet') and 'factor' not in var.op.name and 'Adam' not in var.op.name and 'beta1_power' not in var.op.name and 'beta2_power' not in var.op.name and 'fc_final_sketch46' not in var.op.name and 'global_step' not in var.op.name # count from 0 } snapshot_loader = tf.train.Saver(load_var) print('Firstly training, loaded', pretrained_model) snapshot_loader.restore(sess, pretrained_model) else: raise Exception("No pre-trained model for %s" % mode) else: load_var = {var.op.name: var for var in tf.global_variables() if var.op.name.startswith('ResNet') and 'global_step' not in var.op.name # count from 0 } snapshot_loader = tf.train.Saver(load_var) print('Trained model found, loaded', ckpt.model_checkpoint_path) snapshot_loader.restore(sess, ckpt.model_checkpoint_path) if mode == 'train': log_dir = os.path.join(FLAGS.outputs_base_dir, FLAGS.log_folder_name) os.makedirs(log_dir, exist_ok=True) snapshot_file = os.path.join(snapshot_dir, 'iter_%d.tfmodel') summary_op = tf.summary.merge_all() summary_writer = tf.summary.FileWriter(log_dir, graph=sess.graph) duration_time_n_step = 0 for n_iter in range(FLAGS.max_iteration): start_time = time.time() print('\n#' + str(n_iter)) ## select image index image_idx = random.randint(1, FLAGS.nTrainImgs) ## load images image_name = 'L0_sample' + str(image_idx) + '.png' # e.g. L0_sample5564.png # print("Load:", image_name) image_path = os.path.join(FLAGS.data_base_dir, mode, 'DRAWING_GT', image_name) im = load_image(image_path, mu) # shape = [1, H, W, 3] # print("Ori shape", im.shape) ## load label label_name = 'sample_' + str(image_idx) + '_class.mat' # e.g. sample_1_class.mat label_path = os.path.join(FLAGS.data_base_dir, mode, 'CLASS_GT', label_name) label = load_label(label_path) # shape = [1, H, W], [0, 46] if FLAGS.ignore_class_bg: label = label - 1 # [-1, 45] label[label == -1] = 255 # [0-45, 255] feed_dict = {model.images: im, model.labels: label} _, learning_rate_, global_step, cost, pred, pred_label = \ sess.run([model.train_step, model.learning_rate, model.global_step, model.cost, model.pred, model.pred_label], feed_dict=feed_dict) # print('pred.shape', pred.shape) # (1, H_scale, W_scale, nClasses) print('learning_rate_', learning_rate_) # print('global_step', global_step) print('cost', cost) ## display left time duration_time = time.time() - start_time duration_time_n_step += duration_time if n_iter % FLAGS.count_left_time_freq == 0 and n_iter != 0: left_step = FLAGS.max_iteration - n_iter left_sec = left_step / FLAGS.count_left_time_freq * duration_time_n_step print("Duration_time_%d_step: %s. Left time: %s" % ( FLAGS.count_left_time_freq, str(timedelta(seconds=duration_time_n_step)), str(timedelta(seconds=left_sec)))) duration_time_n_step = 0 ## summary if n_iter % FLAGS.summary_write_freq == 0 and n_iter != 0: summary_str = sess.run(summary_op, feed_dict=feed_dict) summary_writer.add_summary(summary_str, n_iter) summary_writer.flush() ## save model if (n_iter + 1) % FLAGS.save_model_freq == 0 or (n_iter + 1) >= FLAGS.max_iteration: snapshot_saver.save(sess, snapshot_file % (n_iter + 1)) print('model saved to ' + snapshot_file % (n_iter + 1)) print('Training done.') elif mode == 'val' or mode == 'test': def fast_hist(a, b, n): """ :param a: gt :param b: pred """ k = (a >= 0) & (a < n) return np.bincount(n * a[k].astype(int) + b[k], minlength=n ** 2).reshape(n, n) use_dcrf = kwargs['use_dcrf'] eval_base_dir = os.path.join(FLAGS.outputs_base_dir, 'eval_results') os.makedirs(eval_base_dir, exist_ok=True) nImgs = FLAGS.nTestImgs if mode == 'test' else FLAGS.nValImgs colorMap = scipy.io.loadmat(os.path.join(FLAGS.data_base_dir, 'colorMapC46.mat'))['colorMap'] outstr = mode + ' mode\n' cat_max_len = 16 hist = np.zeros((FLAGS.nSketchClasses, FLAGS.nSketchClasses)) for imgIndex in range(1, nImgs + 1): ## load images image_name = 'L0_sample' + str(imgIndex) + '.png' # e.g. L0_sample5564.png image_path = os.path.join(FLAGS.data_base_dir, mode, 'DRAWING_GT', image_name) test_image = load_image(image_path, mu) # shape = [1, H, W, 3] ## load gt_label label_name = 'sample_' + str(imgIndex) + '_class.mat' # e.g. sample_1_class.mat label_path = os.path.join(FLAGS.data_base_dir, mode, 'CLASS_GT', label_name) gt_label = load_label(label_path) # shape = [1, H, W] print('#' + str(imgIndex) + '/' + str(nImgs) + ': ' + image_path) feed_dict = {model.images: test_image, model.labels: 0} pred, pred_label_no_crf = sess.run([model.pred, model.pred_label], feed_dict=feed_dict) if FLAGS.ignore_class_bg: pred_label_no_crf = pred_label_no_crf + 1 # [1, 46] # print('@ pred.shape ', pred.shape) # (1, H, W, nSketchClasses) # print(pred_label_no_crf.shape) # shape = [1, H, W, 1] if use_dcrf: prob_arr = np.squeeze(pred) prob_arr = prob_arr.transpose((2, 0, 1)) # shape = (nSketchClasses, H, W) d_image = np.array(np.squeeze(test_image), dtype=np.uint8) # shape = (H, W, 3) pred_label_crf = seg_densecrf(prob_arr, d_image, nSketchClasses) # shape=[H, W] if FLAGS.ignore_class_bg: pred_label_crf = pred_label_crf + 1 # [1, 46] hist += fast_hist(np.squeeze(gt_label).flatten(), pred_label_crf.flatten(), FLAGS.nSketchClasses) else: hist += fast_hist(np.squeeze(gt_label).flatten(), np.squeeze(pred_label_no_crf).flatten(), FLAGS.nSketchClasses) if imgIndex == nImgs: ## ignore bg pixel with value 0 if FLAGS.ignore_class_bg: hist = hist[1:, 1:] if use_dcrf: print('\nRound', str(imgIndex), ', Use CRF') outstr += '\nRound: ' + str(imgIndex) + ', Use CRF' + '\n' else: print('\nRound', str(imgIndex), ', Not Use CRF') outstr += '\nRound: ' + str(imgIndex) + ', Not Use CRF' + '\n' # overall accuracy acc = np.diag(hist).sum() / hist.sum() print('>>> overall accuracy', acc) outstr += '>>> overall accuracy ' + str(acc) + '\n' # mAcc acc = np.diag(hist) / hist.sum(1) mean_acc = np.nanmean(acc) print('>>> mean accuracy', mean_acc) outstr += '>>> mean accuracy ' + str(mean_acc) + '\n' # mIoU iou = np.diag(hist) / (hist.sum(1) + hist.sum(0) - np.diag(hist)) mean_iou = np.nanmean(iou) print('>>> mean IoU', mean_iou) outstr += '>>> mean IoU ' + str(mean_iou) + '\n' # FWIoU freq = hist.sum(1) / hist.sum() fw_iou = (freq[freq > 0] * iou[freq > 0]).sum() print('>>> freq weighted IoU', fw_iou) print('\n') outstr += '>>> freq weighted IoU ' + str(fw_iou) + '\n' # IoU of each class print('>>> IoU of each class') outstr += '\n>>> IoU of each class' + '\n' for classIdx in range(nSketchClasses): if FLAGS.ignore_class_bg: cat_name = colorMap[classIdx][0][0] else: if classIdx == 0: cat_name = 'background' else: cat_name = colorMap[classIdx - 1][0][0] singlestr = ' >>> ' cat_len = len(cat_name) pad = '' for ipad in range(cat_max_len - cat_len): pad += ' ' singlestr += cat_name + pad + str(iou[classIdx]) print(singlestr) outstr += singlestr + '\n' # write validation result to txt write_path = os.path.join(eval_base_dir, mode + '_results.txt') fp = open(write_path, 'a') fp.write(outstr) fp.close() else: # 'inference' inference_ids = [kwargs['inference_id']] inference_dataset = kwargs['inference_dataset'] black_bg = kwargs['black_bg'] use_dcrf = kwargs['use_dcrf'] colorMap = scipy.io.loadmat(os.path.join(FLAGS.data_base_dir, 'colorMapC46.mat'))['colorMap'] infer_result_base_dir = os.path.join(FLAGS.outputs_base_dir, 'inference_results', inference_dataset) os.makedirs(infer_result_base_dir, exist_ok=True) for img_count, img_id in enumerate(inference_ids): image_name = 'L0_sample' + str(img_id) + '.png' # e.g. L0_sample5564.png image_path = os.path.join(FLAGS.data_base_dir, inference_dataset, 'DRAWING_GT', image_name) infer_image, infer_image_raw = load_image(image_path, mu, return_raw=True) # shape = [1, H, W, 3] / [H, W, 3] print('\n#' + str(img_count + 1) + '/' + str(len(inference_ids)) + ': ' + image_name) feed_dict = {model.images: infer_image, model.labels: 0} pred, pred_label_no_crf, feat_visual \ = sess.run([model.pred, model.pred_label, model.feat_visual], feed_dict=feed_dict) print('@ pred.shape ', pred.shape) # (1, H, W, nSketchClasses) print('@ pred_label_no_crf.shape ', pred_label_no_crf.shape) # shape = [1, H, W, 1], contains [0, nClasses) # print('@ feat_visual.shape ', feat_visual.shape) # shape = (1, 94, 94, 512) if use_dcrf: prob_arr = np.squeeze(pred) prob_arr = prob_arr.transpose((2, 0, 1)) # shape = (nSketchClasses, H, W) d_image = np.array(np.squeeze(infer_image), dtype=np.uint8) # shape = (H, W, 3) pred_label_crf = seg_densecrf(prob_arr, d_image, nSketchClasses) # shape=[H, W], contains [0-46/47] save_base_dir_crf = os.path.join(infer_result_base_dir, 'deeplab_output_crf') os.makedirs(save_base_dir_crf, exist_ok=True) if FLAGS.ignore_class_bg: pred_label_crf += 1 pred_label_crf[infer_image_raw[:, :, 0] != 0] = 0 # [H, W] save_path_crf = os.path.join(save_base_dir_crf, 'sem_result_' + str(img_id) + '.png') visualize_semantic_segmentation(pred_label_crf, colorMap, black_bg=black_bg, save_path=save_path_crf) else: save_base_dir_no_crf = os.path.join(infer_result_base_dir, 'deeplab_output_no_crf') os.makedirs(save_base_dir_no_crf, exist_ok=True) if FLAGS.ignore_class_bg: pred_label_no_crf += 1 pred_label_no_crf = np.squeeze(pred_label_no_crf) pred_label_no_crf[infer_image_raw[:, :, 0] != 0] = 0 # [H, W] save_path_no_crf = os.path.join(save_base_dir_no_crf, 'sem_result_' + str(img_id) + '.png') visualize_semantic_segmentation(pred_label_no_crf, colorMap, black_bg=black_bg, save_path=save_path_no_crf)
def extract_samples_for_inspection(pairs, inter_dir, image_output_dir, seed=123): if not seed is None: np.random.seed(seed) if not os.path.exists(image_output_dir): os.makedirs(image_output_dir, exist_ok=True) raw_id_to_f_mapping = {get_identifier(p[0]): p for p in pairs} fs = os.listdir(inter_dir) # Check existence of identifier file assert 'names.pkl' in fs names = pickle.load(open(os.path.join(inter_dir, 'names.pkl'), 'rb')) for i, n in names.items(): assert get_identifier(n) in raw_id_to_f_mapping # Check phase contrast files phase_contrast_files = [ f for f in fs if f.startswith('X_') and f.endswith('.pkl') ] for i in range(len(phase_contrast_files)): assert 'X_%d.pkl' % i in fs # Sample phase contrast image os.makedirs(os.path.join(image_output_dir, "phase_contrast"), exist_ok=True) random_inds = np.random.choice(list(names.keys()), (50, ), replace=False) for ind in random_inds: file_ind = ind // 100 identifier = get_identifier(names[ind]) try: processed_img = pickle.load( open(os.path.join(inter_dir, 'X_%d.pkl' % file_ind), 'rb'))[ind] raw_img = load_image(raw_id_to_f_mapping[identifier][0]) out_path = os.path.join(image_output_dir, "phase_contrast", "%s.png" % '_'.join(identifier)) save_multi_panel_fig([raw_img, processed_img], out_path) except Exception: print("Error saving sample %s" % '_'.join(identifier)) # try: # # Check discrete segmentation annotations # assert "classify_discrete_labels.pkl" in fs # for i in range(len(phase_contrast_files)): # assert 'segment_discrete_y_%d.pkl' % i in fs # assert 'segment_discrete_w_%d.pkl' % i in fs # classify_discrete_labels = pickle.load(open(os.path.join(inter_dir, "classify_discrete_labels.pkl"), 'rb')) # inds_by_class = {} # for k in classify_discrete_labels: # if classify_discrete_labels[k][0] is None or classify_discrete_labels[k][1] == 0: # continue # label = classify_discrete_labels[k][0] # if not label in inds_by_class: # inds_by_class[label] = [] # inds_by_class[label].append(k) # # Sample discrete fl segmentation (by class) # for cl in inds_by_class: # os.makedirs(os.path.join(image_output_dir, "discrete_segmentation_class_%s" % str(cl)), exist_ok=True) # if len(inds_by_class[cl]) > 20: # random_inds = np.random.choice(list(inds_by_class[cl]), (20,), replace=False) # else: # random_inds = inds_by_class[cl] # for ind in random_inds: # file_ind = ind // 100 # identifier = get_identifier(names[ind]) # try: # raw_pc = load_image(raw_id_to_f_mapping[identifier][0]) # raw_fl = load_image(raw_id_to_f_mapping[identifier][1]) # processed_pc = pickle.load(open(os.path.join(inter_dir, 'X_%d.pkl' % file_ind), 'rb'))[ind] # processed_fl_y = pickle.load(open(os.path.join(inter_dir, 'segment_discrete_y_%d.pkl' % file_ind), 'rb'))[ind] # processed_fl_w = pickle.load(open(os.path.join(inter_dir, 'segment_discrete_w_%d.pkl' % file_ind), 'rb'))[ind] # out_path = os.path.join(image_output_dir, # "discrete_segmentation_class_%s" % str(cl), # "%s.png" % '_'.join(identifier)) # save_multi_panel_fig([raw_pc, # processed_pc, # raw_fl, # None, # processed_fl_y, # processed_fl_w], out_path) # except Exception: # print("Error saving fl(discrete) sample %s" % '_'.join(identifier)) # except Exception: # print("Issue locating discrete segmentation files") try: # Check continuous segmentation annotations assert "classify_continuous_labels.pkl" in fs for i in range(len(phase_contrast_files)): assert 'segment_continuous_y_%d.pkl' % i in fs assert 'segment_continuous_w_%d.pkl' % i in fs classify_continuous_labels = pickle.load( open(os.path.join(inter_dir, "classify_continuous_labels.pkl"), 'rb')) inds_by_class = {} for k in classify_continuous_labels: if classify_continuous_labels[k][ 0] is None or classify_continuous_labels[k][1] == 0: continue label = np.argmax(classify_continuous_labels[k][0]) if not label in inds_by_class: inds_by_class[label] = [] inds_by_class[label].append(k) # Sample continuous fl segmentation (by class) for cl in inds_by_class: os.makedirs(os.path.join( image_output_dir, "continuous_segmentation_class_%s" % str(cl)), exist_ok=True) if len(inds_by_class[cl]) > 20: random_inds = np.random.choice(list(inds_by_class[cl]), (20, ), replace=False) else: random_inds = inds_by_class[cl] for ind in random_inds: file_ind = ind // 100 identifier = get_identifier(names[ind]) try: raw_pc = load_image(raw_id_to_f_mapping[identifier][0]) raw_fl = load_image(raw_id_to_f_mapping[identifier][1]) processed_pc = pickle.load( open(os.path.join(inter_dir, 'X_%d.pkl' % file_ind), 'rb'))[ind] processed_fl_y = pickle.load( open( os.path.join( inter_dir, 'segment_continuous_y_%d.pkl' % file_ind), 'rb'))[ind] processed_fl_y = ( processed_fl_y * np.array([0., 1. / 3, 2. / 3, 1.]).reshape( (1, 1, 4))).sum(2) processed_fl_w = pickle.load( open( os.path.join( inter_dir, 'segment_continuous_w_%d.pkl' % file_ind), 'rb'))[ind] out_path = os.path.join( image_output_dir, "continuous_segmentation_class_%s" % str(cl), "%s.png" % '_'.join(identifier)) save_multi_panel_fig([ raw_pc, processed_pc, raw_fl, None, processed_fl_y, processed_fl_w ], out_path) except Exception: print("Error saving fl(continuous) sample %s" % '_'.join(identifier)) except Exception: print("Issue locating continuous segmentation files")
def run_MURA( batch_size=8, # select a batch of samples to train a time nb_epoch=12, # times of iteration depth=22, # network depth nb_dense_block=4, # number of dense blocks nb_filter=16, # initial number of conv filter growth_rate=12, # numbers of new filters added by each layer dropout_rate=0.2, # dropout rate learning_rate=0.001, # learning rate weight_decay=1E-4, # wight decay plot_architecture=False # plot network architecture ): ################### # Data processing # ################### im_size = 320 # resize images path_train = '/home/yu/Documents/tensorflow/MURA/MURA-v1.1/train/XR_ELBOW' # the absolute path path_valid = '/home/yu/Documents/tensorflow/MURA/MURA-v1.1/valid/XR_ELBOW' X_train_path, Y_train = data_loader.load_path(root_path=path_train, size=im_size) X_valid_path, Y_valid = data_loader.load_path(root_path=path_valid, size=im_size) X_valid = data_loader.load_image(X_valid_path, im_size) # import path for validation Y_valid = np.asarray(Y_valid) nb_classes = 1 img_dim = (im_size, im_size, 1) #tuple channel last ################### # Construct model # ################### # model is one instance of class 'Model' model = densenet.DenseNet(nb_classes, img_dim, depth, nb_dense_block, growth_rate, nb_filter, dropout_rate=dropout_rate, weight_decay=weight_decay) # Model output model.summary() # Build optimizer opt = Adam(lr=learning_rate, beta_1=0.9, beta_2=0.999, epsilon=1e-08) model.compile( loss='binary_crossentropy', optimizer=opt, # optimizer used to update gradient metrics=["accuracy"]) if plot_architecture: from keras.utils import plot_model plot_model(model, to_file='./figures/densenet_archi.png', show_shapes=True) #################### # Network training # #################### print("Start Training") list_train_loss = [] list_valid_loss = [] list_learning_rate = [] best_record = [100, 0, 100, 100] # record the best result start_time = datetime.datetime.now() for e in range(nb_epoch): if e == int(0.25 * nb_epoch): # update learning_rate K.set_value(model.optimizer.lr, np.float32(learning_rate / 10.)) if e == int(0.5 * nb_epoch): K.set_value(model.optimizer.lr, np.float32(learning_rate / 50.)) if e == int(0.75 * nb_epoch): K.set_value(model.optimizer.lr, np.float32(learning_rate / 100.)) split_size = batch_size num_splits = len( X_train_path ) / split_size # Calculate how many batches of training images arr_all = np.arange(len(X_train_path)).astype( int) # Return evenly spaced values within a given interval random.shuffle( arr_all ) # reshuffle, so the order of each training would be different # avoid local optimal solution # with shuffle open, it would be SGD arr_splits = np.array_split( arr_all, num_splits) # Divede the training images to num_splits batches l_train_loss = [] batch_train_loss = [] start = datetime.datetime.now() for i, batch_idx in enumerate( arr_splits): # i: how many batches, batch_idx: each batch X_batch_path, Y_batch = [], [ ] # X_batch_path is the path of images, Y_batch is the label for idx in batch_idx: X_batch_path.append(X_train_path[idx]) Y_batch.append(Y_train[idx]) X_batch = data_loader.load_image( Path=X_batch_path, size=im_size) # load data for training Y_batch = np.asarray( Y_batch ) # Transform the type of Y_batch as array, that is label train_logloss, train_acc = model.train_on_batch( X_batch, Y_batch) # train, return loss and accuracy l_train_loss.append([train_logloss, train_acc]) batch_train_loss.append([train_logloss, train_acc]) if i % 100 == 0: # 100 batches loss_1, acc_1 = np.mean(np.array(l_train_loss), 0) loss_2, acc_2 = np.mean(np.array(batch_train_loss), 0) batch_train_loss = [] print( '[Epoch {}/{}] [Batch {}/{}] [Time: {}] [all_batchs--> train_epoch_logloss: {:.5f}, train_epoch_acc:{:.5f}] ' .format(e + 1, nb_epoch, i, len(arr_splits), datetime.datetime.now() - start, loss_1, acc_1), '[this_100_batchs-->train_batchs_logloss: {:.5f}, train_batchs_acc:{:.5f}]' .format(loss_2, acc_2)) # validate valid_logloss, valid_acc = model.evaluate(X_valid, Y_valid, verbose=0, batch_size=64) list_train_loss.append(np.mean(np.array(l_train_loss), 0).tolist()) list_valid_loss.append([valid_logloss, valid_acc]) list_learning_rate.append(float(K.get_value(model.optimizer.lr))) # to convert numpy array to json serializable print('[Epoch %s/%s] [Time: %s, Total_time: %s]' % (e + 1, nb_epoch, datetime.datetime.now() - start, datetime.datetime.now() - start_time), end='') print( '[train_loss_and_acc:{:.5f} {:.5f}] [valid_loss_acc:{:.5f} {:.5f}]' .format(list_train_loss[-1][0], list_train_loss[-1][1], list_valid_loss[-1][0], list_valid_loss[-1][1])) d_log = {} d_log["batch_size"] = batch_size d_log["nb_epoch"] = nb_epoch d_log["optimizer"] = opt.get_config() d_log["train_loss"] = list_train_loss d_log["valid_loss"] = list_valid_loss d_log["learning_rate"] = list_learning_rate json_file = os.path.join('./log/experiment_log_MURA.json') with open(json_file, 'w') as fp: json.dump(d_log, fp, indent=4, sort_keys=True) record = [ valid_logloss, valid_acc, abs(valid_logloss - list_train_loss[-1][0]), abs(valid_acc - list_train_loss[-1][1]), ] if ((record[0] <= best_record[0]) & (record[1] >= best_record[1])): if e <= int(0.25 * nb_epoch) | (record[2] <= best_record[2]) & ( record[3] <= best_record[3]): best_record = record print('saving the best model:epoch', e + 1, best_record) model.save('save_models/best_MURA_modle@epochs{}.h5'.format(e + 1)) model.save('save_models/MURA_modle@epochs{}.h5'.format(e + 1))