def main(argv): path = argv[1] features = argv[2] data = load_data(path, features) men_vs_women(data) married_women_vs_unmarried(data)
def __init__(self): print config_params self.CLM = CLM_worker(lrate=1., optimizer='adadelta', batch_size=config_params.minibatch, saveto='model.npz', validFreq=2000, dispFreq=100, dropout_input=config_params.CLM_drop_in, dropout_output=config_params.CLM_drop_out, reload_model=config_params.model_dir + '/' + config_params.model_L2S, reload_option=None, log='log1') self.classifier = Classifier( lrate= 1., # Learning rate for sgd (not used for adadelta and rmsprop) optimizer='adadelta', saveto='model.npz', # The best model will be saved there dispFreq= 50, # Display the training progress after this number of updates validFreq= 2000, # Compute the validation error after this number of updates batch_size=config_params. minibatch, # The batch size during training. batch_len_threshold= None, # use dynamic batch size if sequence lengths exceed this threshold valid_batch_size=config_params. minibatch, # The batch size used for validation/test set. lastHiddenLayer=None, dropout_output=config_params.classifier_drop_out, dropout_input=config_params.classifier_drop_in, reload_options= None, # Path to a saved model options we want to start from reload_model=config_params.model_dir + '/' + config_params.model_S2L, embedding= None, # Path to the word embedding file (otherwise randomized) warm_LM=None, logFile='log2' # Path to log file ) self.trainSet, self.validSet, self.testSet = \ load_data(path=config_params.data_dir, n_words=10000, maxlen=None, sort_by_len=True, fixed_valid=True) self.LMscore = numpy.load(config_params.LMScoreFile) self.LMscore = self.LMscore[self.LMscore.files[0]] self.build()
print(pred.shape) residual = pred - data.Y loss = np.linalg.norm(residual, axis=0, ord=2) loss = loss * loss / 0.5 plt.figure() plt.plot(L1, loss, label='Residual term') plt.legend(loc='upper right') plt.xlabel('L1 norm of beta') plt.ylabel('Reconstruction Loss') plt.show() if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('--dir_path', type=str, default='./SGDBeta-V3') args = parser.parse_args() B, L1 = read_beta(args.dir_path) data = load_data() ShowLambda(B, alpha=0.85) ShowL1(B, L1)
def train_model(data_path=data_path, imgs=imgs, msks=msks, tstimgs="", tstmsks="", model_name="model", save_path = "models", num_folds=5, batch_size=32, learning_rate=1e-5, nr_epochs=50, verbosity=1, up=False, start_ch=32, depth=4, inc_rate=2, kernel_size=(3, 3), activation='relu', normalization=None, dropout=0, elastic_deform = None, low_pass = None, high_pass = None, prwt = False, lr_decay = False, lr_schedule=None, model_net = Unet, final_test=False, monitor="val_loss"): ''' DESCRIPTION: Function to load the data, load the model, fit the model and evaluate the model ------- INPUTS: data_path: string, directory of the folder containing the images imgs: string, name of the npy file containing the images msks: string, name of the npy file containing the masks tstimgs: string, name of the npy file containing the test images tstmsks: string, name of the npy file containing the test masks model_name: string, name to identify the model save_path: string, directory of the folder to which to save the results of the callbacks to num_folds: int, number of folds to use for K-fold cross validation batch_size: int, batch size to use in fitting the model learning_rate: float, learning rate to use in compiling the model nr_epochs: int, number of epochs to use in fitting the model verbosity: boolean, whether to print the progress of fitting or not up: boolean, True for using upsampling, False for using Transposed convolution start_ch: int, the number of filters for the first convolutional layers depth: int, the number of convolutional layers inc_rate: number, the factor with which the number of filters is incremented per convolutional layer kernel_size: int or tuple of 2 integers, the kernel size to be used in the convolution layers activation: string, which activation function to use in the convolution layers normalization: function, normalization function. In case of Groupnormalization a tuple of the function and the desired group size dropout: float between 0-1, the dropout rate to be used elastic_deform: None or tuple of 2 numbers, (alpha, sigma) with alpha being the scaling factor for the transformation and sigma being the standard deviation for the gaussian convolution low_pass: None or int, giving the standard deviation used for the gaussian low-pass filter applied to the images high_pass: None or int, giving the standard deviation used for the gaussian high_pass filter applied to the images prwt: boolean, whether to apply a prewitt filter to the images or not lr_decay: boolean, whether to use a scheduled learning rate using the schedule from 'Model.py' or not lr_schedule: function, the schedule to be used for the learning rate model_net: function, which model architecture to use (from Model.py: Unet or Mnet) final_test: boolean, stating if the model should be optimized (k-fold = 5 folds, validation split is used, no test data is needed) or if the model performance should be tested (in 10 fold, training on all data en testing on test data) monitor: string, which output of the model to monitor by the callbacks, in case final_test=True it will automaticly be set to "loss" ------- OUTPUTS: A .h5 file per fold containing the model with its trained weights A .out file per fold containing the log of the training process in CSV format A .csv file where each row contains the DSC and train time per fold of the model, if there is already a file present the results are appended A .csv file where each row contains the mean and standard deviation of the DSCs and times of all folds of a model, if there is already a file present the results are appended ''' ##### load data and optional test data ##### images, masks = load_data(data_path, imgs, msks, low_pass=low_pass, high_pass=high_pass, prwt=prwt) if final_test: print_func('Test Data:') test_images, test_masks = load_data(data_path, tstimgs, tstmsks, low_pass=low_pass, high_pass=high_pass, prwt=prwt) if model_net == Mnet: test_masks = downsample_image(test_masks, depth-1) #monitor = "loss" num_folds = 10 ##### save arguments for the model to dictionairy ##### arg_dict_model = {"start_ch": start_ch, "depth": depth, "inc_rate": inc_rate, "kernel_size": kernel_size, "activation": activation, "normalization": normalization, "dropout": dropout, "learning_rate": learning_rate, "up": up} ##### prepare for k-fold cross validation ##### kfold = KFold(n_splits=num_folds, shuffle=True) fold_no = 1 dice_per_fold, time_per_fold = [], [] for train, val in kfold.split(images, masks): print_func(f'Training for fold {fold_no} (of {num_folds}) ... \nModel name: {model_name}') train_im, train_msk, val_im, val_msk = images[train], masks[train], images[val], masks[val] if elastic_deform is not None: train_im, train_msk = image_transformation(train_im, train_msk, elastic_deform) if model_net == Mnet: #M-net has multiple outputs, the train and validation masks are downsampled to match the outputs of the model print_func("prepare data for Mnet") train_msk = downsample_image(train_msk, depth-1) val_msk = downsample_image(val_msk, depth-1) ##### load model with random initialized weights ###### model = model_net(**arg_dict_model) ##### load callbacks ##### save_dir = save_path + '/' + model_name + " K_" + str(fold_no) callbacks_list = [] callbacks_list.append(ModelCheckpoint(save_dir + ' weights.h5', monitor=monitor, save_best_only=True)) callbacks_list.append(CSVLogger(os.path.join(save_dir + ' log.out'), append=True, separator=';')) callbacks_list.append(EarlyStopping(monitor = monitor, verbose = 1, min_delta = 0.0001, patience = 5, mode = 'auto', restore_best_weights = True)) if lr_decay: lr_sched = LearningRateScheduler(schedule, verbose = 1) if lr_schedule is None else LearningRateScheduler(lr_schedule, verbose = 1) callbacks_list.append(lr_sched) ##### fit model ##### arg_dict_fit = {"x": train_im, "y": train_msk, "validation_data": (val_im, val_msk), "batch_size": batch_size, "epochs": nr_epochs, "verbose": verbosity, "shuffle": True} start_time = time() model.fit(callbacks=callbacks_list, **arg_dict_fit) train_time = int(time()-start_time) if final_test: val_im, val_msk = test_images, test_masks ##### evaluate model ##### if model_net == Mnet: scores = eval_Mnet(val_im, val_msk['o1'], model, verbose=1) else: scores = model.evaluate(val_im, val_msk, verbose=0) ##### save scores of fold ##### print_func(f"Scores \nDice: {scores[1]} \nTime: {train_time}") dice_per_fold.append(scores[1]) time_per_fold.append(train_time) save_results(model_name + f' K_{fold_no}', scores[1], train_time) fold_no += 1 ##### save scores of model ##### save_results(model_name, dice_per_fold, time_per_fold, False)
def post_process(model_file, weights_path, data_path, imgs, msks, model_name = "", n = None, m = False, low_pass = None, high_pass = None, threshold=0.95, disk_size=5, smooth_sigma=1, smooth_trsh=0.5): ''' DESCRIPTION: Function to test and apply post-processing techniques on trained models ------- INPUTS: model_file: string, directory of the .json file containing the model architecture weights_path: string, directory of the .h5 files containing the trained weights of the model data_path: string, directory of the folder containing the images imgs: string, name of the npy file containing the images msks: string, name of the npy file containing the masks model_name: string, name to identify the model n: None or int, the number of images to predict and test. If None all images given are tested m: boolean, whether the data should be prepared for an M-net architecture or not low_pass: None or int, giving the standard deviation used for the gaussian low-pass filter applied to the images high_pass: None or int, giving the standard deviation used for the gaussian high_pass filter applied to the images threshold: float between 0 and 1, the threshold to apply to the masks disk_size: int, the radius of the disk shaped structuring element used in opening by reconstruction smooth_sigma: int, the standard deviation used for the gaussian low-pass filter for smoothing edges smooth_trsh: float between 0 and 1, the threshold to apply after low-pass filtering ------- OUTPUTS: A .csv file where each row contains the mean and standard deviation of the DSCs of all weights of a model per post-processing option, if there is already a file present the results are appended ''' images, masks = load_data(data_path, imgs, msks, low_pass=low_pass, high_pass=high_pass) if n is None: n = len(images) print_func("load model") json_file = open(model_file, 'r') loaded_model_json = json_file.read() json_file.close() model = model_from_json(loaded_model_json) weights = os.listdir(weights_path) weights = [weight for weight in weights if weight.find(".h5")>-1] scores = pd.DataFrame() for weight in weights: print_func(f"load weigths: {weight}") model.load_weights(os.path.join(weights_path, weight)) predictions = model.predict(images[:n], verbose=1) if m: predictions = sum(predictions)/len(predictions) sc0 = np.mean(list(map(dice_coef_pred, masks[:n], predictions))) if threshold is not None: pp1, sc1 = [], [] if type(threshold) != list: threshold = [threshold] for trsh in threshold: pp1.append(post_process_thresholding(copy.copy(predictions), trsh)) for res in pp1: sc1.append(np.mean(list(map(dice_coef_pred, masks[:n], res)))) max_idx1 = sc1.index(max(sc1)) else: pp1, sc1, threshold, max_idx1 = [predictions], [], [None], 0 if disk_size is not None: pp2, sc2 = [], [] if type(disk_size) != list: disk_size = [disk_size] for ds in disk_size: pp2.append(post_process_openingbyreconstruction(copy.copy(pp1[max_idx1]), ds)) for res in pp2: sc2.append(np.mean(list(map(dice_coef_pred, masks[:n], res)))) max_idx2 = sc2.index(max(sc2)) else: pp2, sc2, disk_size, max_idx2 = pp1, [], [None], 0 if smooth_sigma is not None: pp3, sc3 = [], [] if type(smooth_sigma) != list: smooth_sigma = [smooth_sigma] if type(smooth_trsh) != list: smooth_trsh = [smooth_trsh] for ss in smooth_sigma: for st in smooth_trsh: pp3.append(post_process_smoothingedges(copy.copy(pp2[max_idx2]), ss, st)) for res in pp3: sc3.append(np.mean(list(map(dice_coef_pred, masks[:n], res)))) else: sc3 = [] scores = scores.append([[sc0, *sc1, *sc2, *sc3]], ignore_index=True) for key, value in scores.iteritems(): m_name = model_name + '.' if key>0 and key<=len(threshold): m_name = m_name + "trsh" + str(threshold[key-1]) elif key>len(threshold) and key<=len(threshold)+len(disk_size): m_name = m_name + "disk" + str(disk_size[key-1-len(threshold)]) elif key>len(threshold)+len(disk_size):m_name + "smooth_sigma" + str(key-1-len(threshold)-len(disk_size)) save_results(m_name, value, 0, False)
def mask_parse(mask): mask = np.squeeze(mask) mask = [mask, mask, mask] mask = np.transpose(mask, (1, 2, 0)) return mask if __name__ == "__main__": np.random.seed(42) tf.random.set_seed(42) PATH = "../../Dataset/" BATCH = 8 (train_x, train_y), (valid_x, valid_y), (test_x, test_y) = load_data(PATH) test_dataset = tf_dataset(test_x, test_y, batch=BATCH) test_steps = len(test_x) // BATCH if len(test_x) % BATCH != 0: test_steps += 1 with CustomObjectScope({'iou': iou}): model = tf.keras.models.load_model("../Models/model_transversal.h5") model.evaluate(test_dataset, steps=test_steps) for i, (x, y) in tqdm(enumerate(zip(test_x, test_y)), total=len(test_x)): x = read_image(x) y = read_mask(y) y_pred = model.predict(np.expand_dims(x, axis=0))
def main(): parser = argparse.ArgumentParser() parser.add_argument( '--device', '-d', type=int, default=0, help='Using which gpu?(Should be one of 0-7; Default:0)') parser.add_argument('--lamabda', type=float, default=1.0) parser.add_argument('--save_path', type=str, default='./SGDBeta-V2/') parser.add_argument('--continue_beta', type=str, default='./lambda.2000.0.txt') args = parser.parse_args() DEVICE = torch.device('cuda', args.device) save_path = args.save_path TOTAL_ITER = 200000 InitiaLearningRate = 1e-5 / (1 + args.lamabda / 20) affine = Affine() if args.continue_beta is not None and os.path.isfile(args.continue_beta): print('Warm starting from {}'.format(args.continue_beta)) beta = np.loadtxt(args.continue_beta).astype(np.float32)[np.newaxis, :] affine.fc.weight = torch.nn.Parameter(torch.tensor(beta)) TOTAL_ITER = TOTAL_ITER // 10 InitiaLearningRate = InitiaLearningRate / 2 GetLearningRate = LinearLearningRatePolicy(TOTAL_ITER, InitiaLearningRate) else: GetLearningRate = LinearLearningRatePolicy(TOTAL_ITER, InitiaLearningRate) affine = affine.to(DEVICE) criterion = nn.MSELoss(reduce=True, size_average=False).to(DEVICE) optimizer = optim.SGD(affine.parameters(), lr=GetLearningRate(0), momentum=0.8) numpy_data = load_data() X = torch.from_numpy(numpy_data.X).type(torch.FloatTensor).to(DEVICE) Y = torch.tensor(numpy_data.Y).type(torch.FloatTensor).to(DEVICE) beta = torch.tensor(numpy_data.beta).type(torch.FloatTensor).to(DEVICE) data = Data(X, Y, beta) pbar = tqdm(range(TOTAL_ITER)) print('Lambda:{} -- Total Iters:{} -- Initial Lr:{}'.format( args.lamabda, TOTAL_ITER, InitiaLearningRate)) for cur_iter in pbar: residual_loss, l1 = train_one_iter(data, criterion, args.lamabda, optimizer, affine) pbar.set_postfix(residual_loss='{:.2f}'.format(residual_loss), l1='{:.2f}'.format(l1)) beta = affine.fc.weight.detach().cpu().numpy() np.savetxt(save_path, beta)