def save_model_outputs(model, _dataset, model_path): npy_path = model_path + 'npy/' create_folders(npy_path, model_path + 'model/') model_soft = Model(model.input, model.get_layer('features').output) # save softmax predictions pred = model.predict(_dataset.x_train)[:, :_dataset.num_classes] pred_int = np.argmax(pred, axis=1) np.save(npy_path + 'train_preds.npy', pred) np.save(npy_path + 'train_preds_int.npy', pred_int) pred = model.predict(_dataset.x_test)[:, :_dataset.num_classes] pred_int = np.argmax(pred, axis=1) np.save(npy_path + 'test_preds.npy', pred) np.save(npy_path + 'test_preds_int.npy', pred_int) # save logits logits_train = model_soft.predict( _dataset.x_train)[:, :_dataset.num_classes] logits_test = model_soft.predict(_dataset.x_test)[:, :_dataset.num_classes] np.save(npy_path + 'train_logits.npy', logits_train) np.save(npy_path + 'test_logits.npy', logits_test) # save confusion matrices cm_train = plot_cm(model, _dataset.x_train, _dataset.y_train_int(), _dataset.class_names, model_path + 'train_cm.png') cm_test = plot_cm(model, _dataset.x_test, _dataset.y_test_int(), _dataset.class_names, model_path + 'test_cm.png') np.save(npy_path + 'train_cm.npy', cm_train) np.save(npy_path + 'test_cm.npy', cm_test) # save distance matrices plot_dm(model_soft, _dataset.x_train, _dataset.y_train_int(), _dataset.class_names, model_path + 'train_dm.png') plot_dm(model_soft, _dataset.x_test, _dataset.y_test_int(), _dataset.class_names, model_path + 'test_dm.png') # save model plot_model(model, model_path + 'model/model.png') save_model(model, model_path + 'model/model') K.clear_session()
def prep_noisylabels(dataset, folders, noise_type, noise_ratio, verbose, alpha, temperature, is_dropout): # prepare required models first prep_teacher_model(dataset, verbose) prep_student(dataset, verbose, alpha, temperature) # generate and save corrupted labels for each noise type for given noise ratio _dataset = get_data(dataset) # copy xy model as none for baseline path = str(Path(folders['logdir']).parent) if not os.path.isdir(path + '/none/'): shutil.copytree('{}/models/teacher'.format(dataset), path + '/none/') # generate noisy labels y_train_noisy, y_test_noisy, probs = get_noisy_labels( _dataset, noise_type, noise_ratio) if PLOT_NOISE: y_train_clean, y_test_clean = _dataset.y_train_int( ), _dataset.y_test_int() create_folders(folders['noisedir']) if not isfile(folders['noisedir'] + 'cmtest.png'): print('Noise for {} doesnt exist, creating it...'.format( folders['noisedir'])) # plot confused samples if probs is not None: create_folders(folders['noisedir'] + '/plots') np.save(folders['noisedir'] + 'probs.npy', probs) _dataset_noisy = get_data(dataset, y_noisy=y_train_noisy, y_noisy_test=y_test_noisy) plot_confused_samples(probs, _dataset_noisy, path=folders['noisedir'] + 'plots/') plot_confused_samples2(probs, _dataset_noisy, path=folders['noisedir'] + 'plots/') # save confusion matrix cm = confusion_matrix(y_train_clean, y_train_noisy) plot_matrix(cm, _dataset.class_names, title='Noise ratio: {}'.format(noise_ratio)) plt.savefig(folders['noisedir'] + 'cmtrain.png') cm = confusion_matrix(y_test_clean, y_test_noisy) plot_matrix(cm, _dataset.class_names, title='Noise ratio: {}'.format(noise_ratio)) plt.savefig(folders['noisedir'] + 'cmtest.png') return y_train_noisy, y_test_noisy
def tensor_board(dataset, log_dir, is_embed=False, **kwargs): tb_dir = log_dir + 'tensorboard/' create_folders(tb_dir) with open(join(tb_dir, 'metadata.tsv'), 'w') as f: np.savetxt(f, dataset.y_test_int()) if not is_embed: return TensorBoard(log_dir=tb_dir, **kwargs) else: return TensorBoard(log_dir=tb_dir, embeddings_freq=1, embeddings_layer_names=['features'], embeddings_metadata='metadata.tsv', embeddings_data=dataset.x_test, **kwargs)
def train(dataset, model, epochs=50, batch_size=128, log_dir=None, callbacks=[], verbose=1): # prepare folders create_folders(log_dir + 'model/', log_dir + 'npy/') # get data x_train, y_train, x_test, y_test = dataset.get_data() x_train, x_val, y_train, y_val = train_test_split(x_train, y_train, test_size=0.5, random_state=RANDOM_SEED) # callbacks mycallbacks = [] mycallbacks.extend(callbacks) if log_dir is not None: mycallbacks.extend([ MyLogger(log_dir, dataset), learning_rate_scheduler(lr_schedule, verbose=verbose), lr_plateau(factor=np.sqrt(0.1), cooldown=0, patience=5, min_lr=0.5e-6, verbose=verbose) ]) plot_model(model, log_dir + 'model/model.png') # image generator datagen = ImageDataGenerator(width_shift_range=0.1, height_shift_range=0.1) # train model on clean data model.fit_generator(datagen.flow(x_train, y_train, batch_size=batch_size), steps_per_epoch=dataset.num_train_samples / batch_size, epochs=epochs, validation_data=(x_val, y_val), verbose=verbose, callbacks=mycallbacks) loss, acc = model.evaluate(x_test, y_test, verbose=0) print('val_loss:', loss, '- val_acc:', acc) del callbacks gc.collect() return model
def train_coteaching(dataset, model1, model2, epochs=50, batch_size=128, log_dir=None, forget_rate=0.2, num_graduals=10, exponent=0.2, learning_rate=1e-3, epoch_decay_start=30): def epoch_stats(dataset, model, epoch, logdir, csv_path=None): x_train, y_train_noisy, x_test, y_test = dataset.get_data() y_train_int = dataset.y_noisy_int() y_test_int = dataset.y_test_int() clean_index = dataset.idx_clean noisy_index = dataset.idx_noisy if csv_path is not None: train_loss, train_acc = model.evaluate(x_train, y_train, verbose=0) test_loss, test_acc = model.evaluate(x_test, y_test, verbose=0) print("-%s: acc_test: %.4f - acc_mix: %.4f" % (model.name, test_acc, train_acc)) df = pd.read_csv(log_dir + csv_path) #row = [{'epoch':epoch, 'acc':acc_mix,'loss':loss_mix, 'val_acc':acc_test, 'val_loss':loss_test,'test_acc':test_acc,'test_loss':test_loss}] row = [{ 'epoch': epoch, 'acc': train_acc, 'loss': train_loss, 'test_acc': test_acc, 'test_loss': test_loss }] df = df.append(row) df.to_csv(log_dir + csv_path, index=False) xticks = np.arange(1, epoch + 1, 1.0) plt.figure() plt.plot(xticks, df['acc'], label="acc") #plt.plot(xticks, df['val_acc'], label="val_acc") plt.plot(xticks, df['test_acc'], label="test_acc") plt.legend(loc='best') plt.xlabel('# iterations') plt.xticks(xticks) plt.title('Accuracy') plt.savefig(log_dir + 'accuracy_{}.png'.format(model.name)) plt.clf() plt.plot(xticks, df['loss'], label="loss") #plt.plot(xticks, df['val_loss'], label="val_loss") plt.plot(xticks, df['test_loss'], label="test_loss") plt.legend(loc='best') plt.xlabel('# iterations') plt.xticks(xticks) plt.title('Loss') plt.savefig(log_dir + 'loss_{}.png'.format(model.name)) plt.close() create_folders(log_dir + 'model/', log_dir + 'npy/') # get data x_train, y_train, x_test, y_test = dataset.get_data() model1._name = 'model1' model2._name = 'model2' # number of batches in an epoch num_batch_iter = x_train.shape[0] / batch_size # calculate forget rates for each epoch (from origianl code) forget_rates = np.ones(epochs) * forget_rate forget_rates[:num_graduals] = np.linspace(0, forget_rate**exponent, num_graduals) # calculate learning rates for each epoch (from origianl code) learning_rates = [learning_rate] * epochs for i in range(epoch_decay_start, epochs): learning_rates[i] = float(epochs - i) / ( epochs - epoch_decay_start) * learning_rate if log_dir is not None: #logcsv_cols =['epoch','acc','loss','val_acc','val_loss','test_acc','test_loss'] logcsv_cols = ['epoch', 'acc', 'loss', 'test_acc', 'test_loss'] df = pd.DataFrame(columns=logcsv_cols) df.to_csv(log_dir + 'log1.csv', index=False) df.to_csv(log_dir + 'log2.csv', index=False) for e in range(1, epochs): # if learning rate changes, recompile if (learning_rates[e] != learning_rates[e - 1]): model1.compile(loss='categorical_crossentropy', optimizer=SGD(lr=learning_rates[e], decay=1e-6, momentum=0.9, nesterov=True), metrics=['accuracy']) model2.compile(loss='categorical_crossentropy', optimizer=SGD(lr=learning_rates[e], decay=1e-6, momentum=0.9, nesterov=True), metrics=['accuracy']) #model2.compile(loss='categorical_crossentropy', optimizer=Adam(learning_rates[e]), metrics=['accuracy']) remember_rate = 1 - forget_rates[e] print("Epoch: %d/%d; Learning rate: %.7f; n_keep: %d" % (e + 1, epochs, learning_rates[e], remember_rate * batch_size)) # iterate for each batch in an epoch for (i, (x_batch, y_batch)) in enumerate(dataset.flow_train(batch_size)): num_remember = int(remember_rate * len(x_batch)) # select samples based on model 1 y_pred = model1.predict_on_batch(x_batch) cross_entropy = np.sum(-y_batch * np.log(y_pred + 1e-8), axis=1) batch_idx1 = np.argsort(cross_entropy)[:num_remember] # select samples based on model 2 y_pred = model2.predict_on_batch(x_batch) cross_entropy = np.sum(-y_batch * np.log(y_pred + 1e-8), axis=1) batch_idx2 = np.argsort(cross_entropy)[:num_remember] # training model1.train_on_batch(x_batch[batch_idx2, :], y_batch[batch_idx2, :]) model2.train_on_batch(x_batch[batch_idx1, :], y_batch[batch_idx1, :]) if i >= num_batch_iter: break epoch_stats(dataset, model1, e, log_dir, 'log1.csv') epoch_stats(dataset, model2, e, log_dir, 'log2.csv') # choose best model loss1, acc1 = model1.evaluate(x_test, y_test, verbose=0) loss2, acc2 = model2.evaluate(x_test, y_test, verbose=0) if acc1 > acc2: os.rename(log_dir + 'log1.csv', log_dir + 'log.csv') return model1 else: os.rename(log_dir + 'log2.csv', log_dir + 'log.csv') return model2
def main(dataset_name, model_name, epochs, batch_size, noise_type, noise_ratio, verbose=1, alpha=util.ALPHA, temperature=16, is_dropout=False, percentage=1): K.clear_session() os.environ['CUDA_VISIBLE_DEVICES'] = '0' seed_everything() # folders to be used noise_base_path = 'nr_{}'.format( noise_ratio) if not is_dropout else 'nr_{}_do'.format(noise_ratio) folders = { 'logbase': '{}/logs_{}/'.format(dataset_name, percentage), 'logbase_nr': '{}/logs_{}/{}/{}/'.format(dataset_name, percentage, noise_base_path, model_name), 'logdir': '{}/logs_{}/{}/{}/{}/'.format(dataset_name, percentage, noise_base_path, model_name, noise_type), 'modelbase': '{}/models/'.format(dataset_name), 'noisebase': '{}/noisylabels/'.format(dataset_name), 'noisedir': '{}/noisylabels/{}/'.format(dataset_name, noise_type), 'dataset': '{}/dataset'.format(dataset_name) } # if log file already exis"ts dont run it again if isfile(folders['logdir'] + 'model/model.h5') and isfile(folders['logdir'] + 'model/model.json'): print('Logs exists, skipping run...') return # clean empty logs if there is any clean_empty_logs() # create necessary folders create_folders(folders['dataset'], folders['logdir']) # generate noisy labels y_train_noisy, y_test_noisy = prep_noisylabels(dataset_name, folders, noise_type, noise_ratio, verbose, alpha, temperature, is_dropout) # load dataset with noisy labels dataset = get_data(dataset_name, y_noisy=y_train_noisy, y_noisy_test=y_test_noisy) dataset.get_percentage(percentage) # stats before training print( 'Dataset: {}, model: {}, noise_type: {}, noise_ratio: {}, epochs: {}, batch: {} , dropout: {}' .format(dataset.name, model_name, noise_type, noise_ratio, epochs, batch_size, is_dropout)) dataset.get_stats() dataset.save_cm_train(folders['logdir'] + 'corrupted_data.png') # train model if model_name == 'coteaching': model1 = get_model(dataset, model_name, is_dropout=is_dropout) model2 = get_model(dataset, model_name, is_dropout=is_dropout) model = train_coteaching(dataset, model1, model2, epochs, batch_size, folders['logdir']) else: #cm = np.load('{}/models/xy/npy/test_cm.npy'.format(dataset_name)) cm = dataset.get_cm_train() cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis] model = get_model(dataset, model_name, cm, is_dropout=is_dropout) model = train(dataset, model, epochs, batch_size, folders['logdir'], verbose=verbose) # performance analysis postprocess(dataset, model, noise_type, noise_ratio, folders, y_test_noisy) K.clear_session()
def model_checkpoint(log_dir, **kwargs): logdir = log_dir + 'model/' create_folders(logdir) return ModelCheckpoint(logdir + "checkpoint.hdf5", **kwargs)