def test_acc_last_epoch(model_list, dataset='mnist', num_classes=10, noise_ratio=10, epochs=50): """ Test acc throughout training. """ print('Dataset: %s, epochs: %s, noise ratio: %s%%' % (dataset, epochs, noise_ratio)) # load data _, _, X_test, Y_test = get_data(dataset) # convert class vectors to binary class matrices Y_test = to_categorical(Y_test, num_classes) # load model image_shape = X_test.shape[1:] model = get_model(dataset, input_tensor=None, input_shape=image_shape) sgd = SGD(lr=0.01, momentum=0.9) for model_name in model_list: # the critical sample ratio of the representations learned at every epoch model_path = 'model/%s_%s_%s.hdf5' % (model_name, dataset, noise_ratio) model.load_weights(model_path) model.compile(loss=cross_entropy, optimizer=sgd, metrics=['accuracy']) _, test_acc = model.evaluate(X_test, Y_test, batch_size=128, verbose=0) print('model: %s, epoch: %s, test_acc: %s' % (model_name, epochs - 1, test_acc))
def learn_nn(dataset_name, targets, batch_size, epochs): for target in targets: _, X, Y = dt.get_data(target=target, dataset_name=dataset_name) # Модель model = md.get_simple_nn(X.shape[1]) if dataset_name == 'full harding': non_cat_title = tl.full_hard_non_cat_title cat_title = tl.full_hard_cat_title elif dataset_name == 'double harding': non_cat_title = tl.double_hard_non_cat_title cat_title = tl.double_hard_cat_title else: raise ValueError # Обучение sc_data = X[non_cat_title] scaler = preprocessing.StandardScaler() sc_data = scaler.fit_transform(sc_data) sc_data = pd.DataFrame(sc_data) ct_data = X[cat_title] X = sc_data.combine_first(ct_data).values Y = Y.values model.fit(X, Y, epochs=epochs, batch_size=batch_size, verbose=1) save_model(dataset_name, model, target, scaler, non_cat_title, cat_title)
def get_validation_function(args, model): class ValidationFunction: def __init__(self, func, iterations_per_epoch): self.func = func self.validation_iterations_per_epoch = iterations_per_epoch if args.validation_mode == "none": return None if args.use_popdist and args.popdist_rank != 0: # Run validation in a single process. return None if args.mixup_enabled or args.cutmix_enabled: assert isinstance(model, augmentations.AugmentationModel) model = model.model opts = create_validation_opts(args, use_popdist=args.use_popdist) test_data = datasets.get_data(args, opts, train=False, async_dataloader=True, return_remaining=True) inference_model = poptorch.inferenceModel(model, opts) def validation_func(): model.eval() if inference_model._executable: inference_model.attachToDevice() val_acc = test(inference_model, test_data) inference_model.detachFromDevice() model.train() return val_acc return ValidationFunction(validation_func, len(test_data))
def get_features_types(): X, y = get_data('../data/trainDF.csv') dtypes = pd.DataFrame(X.dtypes.rename('type')).reset_index().astype('str') numeric = dtypes[(dtypes.type.isin(['int64', 'float64']))]['index'].values categorical = dtypes[~(dtypes['index'].isin(numeric)) & (dtypes['index'] != 'y')]['index'].values return categorical, numeric
def run_GB_learning(dataset_name, n_trees, learning_rate, max_depth): for target in targets: _, X, Y = dt.get_data(target=target, dataset_name=dataset_name) # Модель model = ensemble.GradientBoostingRegressor(n_estimators=n_trees, learning_rate=learning_rate, max_depth=max_depth) # Обучение if dataset_name == 'full harding': non_cat_title = tl.full_hard_non_cat_title cat_title = tl.full_hard_cat_title elif dataset_name == 'double harding': non_cat_title = tl.double_hard_non_cat_title cat_title = tl.double_hard_cat_title else: raise ValueError sc_data = X[non_cat_title] scaler = preprocessing.StandardScaler() sc_data = scaler.fit_transform(sc_data) sc_data = pd.DataFrame(sc_data) ct_data = X[cat_title] X = sc_data.combine_first(ct_data).values Y = Y.values model.fit(X, Y) save_model(dataset_name, model, target, scaler, non_cat_title, cat_title)
def validate_checkpoints(checkpoint_list, test_data=None): checkpoint = torch.load(checkpoint_list[0]) opts = checkpoint['opts'] utils.Logger.setup_logging_folder(opts) # make sure the order is ascending def ckpt_key(ckpt): return int(ckpt.split('_')[-1].split('.')[0]) try: checkpoint_list = sorted(checkpoint_list, key=ckpt_key) except: logging.warn("Checkpoint names are changed, which may cause inconsistent order in evaluation.") model_opts = create_validation_opts(opts) if test_data is None: logging.info("Loading the data") test_data = datasets.get_data(opts, model_opts, train=False, async_dataloader=True) logging.info("Create model") model = models.get_model(opts, datasets.datasets_info[opts.data], pretrained=False) model.eval() inference_model = poptorch.inferenceModel(model, model_opts) for checkpoint in checkpoint_list: load_checkpoint_weights(inference_model, checkpoint) acc = test(inference_model, test_data, opts) epoch_nr = torch.load(checkpoint)["epoch"] result_dict = {"validation_epoch": epoch_nr, "validation_iteration": opts.logs_per_epoch * epoch_nr, "validation_accuracy": acc} utils.Logger.log_validate_results(result_dict)
def lid_trend_through_training(model_name='ce', dataset='mnist', noise_type='sym', noise_ratio=0.): """ plot the lid trend for clean vs noisy samples through training. This can provide some information about manifold learning dynamics through training. """ print('Dataset: %s, noise type: %s, noise ratio: %.1f' % (dataset, noise_type, noise_ratio)) lids, acc_train, acc_test = None, None, None # get LID of raw inputs lid_subset = 128 k = 20 X_train, Y_train, X_test, Y_test = get_data(dataset) rand_idxes = np.random.choice(X_train.shape[0], lid_subset * 10, replace=False) X_train = X_train[rand_idxes] X_train = X_train.reshape((X_train.shape[0], -1)) lid_tmp = [] for i in range(10): s = i * 128 e = (i + 1) * 128 lid_tmp.extend(mle_batch(X_train[s:e], X_train[s:e], k=k)) lid_X = np.mean(lid_tmp) print('LID of input X: ', lid_X) # load pre-saved to avoid recomputing lid_saved = "log/lid_%s_%s_%s%s.npy" % (model_name, dataset, noise_type, noise_ratio) acc_saved = "log/acc_%s_%s_%s%s.npy" % (model_name, dataset, noise_type, noise_ratio) if os.path.isfile(lid_saved): lids = np.load(lid_saved) lids = np.insert(lids, 0, lid_X) print(lids) if os.path.isfile(acc_saved): data = np.load(acc_saved) acc_train = data[0][:] acc_test = data[1][:] acc_train = np.insert(acc_train, 0, 0.) acc_test = np.insert(acc_test, 0, 0.) plot(model_name, dataset, noise_ratio, lids, acc_train, acc_test)
def run_model(dataset_name, model_name, model, plot, target, n_splits, is_plot=False): ns, x, y = dt.get_data(target=target, dataset_name=dataset_name) # Обучение kfold = StratifiedKFold(n_splits=n_splits, shuffle=True) cvscores = [] cat_title, non_cat_title = get_titles_lists(dataset_name) it = 0 for train, test in kfold.split(x, y): scaler = preprocessing.StandardScaler() x_train, y_train = get_x_y_data(x, y, train, scaler, non_cat_title, cat_title) x_test, y_test = get_x_y_data(x, y, test, scaler, non_cat_title, cat_title) model.fit(x_train.values, y_train) # Тестирование predicted = model.predict(x_test.values) scores = metrics.mean_squared_error(y_test, predicted) print('Error: {:.2f}'.format(scores)) cvscores.append(scores) print('Corr: {:.2f}\n'.format(np.corrcoef(y_test, predicted)[0][1])) save_histogram(predicted, y_test, it) x_test_sc = pd.DataFrame(scaler.inverse_transform( x_test[non_cat_title]), columns=non_cat_title, index=x_test.index) x_test_sc[target] = y_test x_test_sc[target + u'(сеть)'] = predicted x_test_sc.combine_first( pd.DataFrame(ns)).dropna().to_excel(dp.results[model_name] + '/res{}.xlsx'.format(str(it))) it += 1 if is_plot: plot(model, x_test, y_test) print("%.2f (+/- %.2f)" % (np.mean(cvscores), np.std(cvscores))) return [np.mean(cvscores), np.std(cvscores)]
def validate_checkpoints(checkpoint_list, test_data=None): checkpoint = torch.load(checkpoint_list[0]) args = checkpoint['args'] utils.Logger.setup_logging_folder(args) # make sure the order is ascending def ckpt_key(ckpt): return int(ckpt.split('_')[-1].split('.')[0]) try: checkpoint_list = sorted(checkpoint_list, key=ckpt_key) except: logging.warn( "Checkpoint names are changed, which may cause inconsistent order in evaluation." ) # Validate in a single instance opts = create_validation_opts(args, use_popdist=False) args.use_popdist = False args.popdist_size = 1 if test_data is None: test_data = datasets.get_data(args, opts, train=False, async_dataloader=True, return_remaining=True) model = models.get_model(args, datasets.datasets_info[args.data], pretrained=False) model.eval() # Load the weights of the first checkpoint for the model models.load_model_state_dict(model, checkpoint['model_state_dict']) inference_model = poptorch.inferenceModel(model, opts) for checkpoint in checkpoint_list: if inference_model.isCompiled(): load_checkpoint_weights(inference_model, checkpoint) val_accuracy = test(inference_model, test_data) epoch_nr = torch.load(checkpoint)["epoch"] log_data = { "validation_epoch": epoch_nr, "validation_iteration": epoch_nr * len(test_data), "validation_accuracy": val_accuracy, } utils.Logger.log_validate_results(log_data)
def get_validation_function(opts, model): if run_opts.validation_mode == "none": return None inference_model_opts = create_validation_opts(opts) test_data = datasets.get_data(opts, inference_model_opts, train=False, async_dataloader=True) inference_model = poptorch.inferenceModel(model, inference_model_opts) def validation_func(): model.eval() if inference_model._executable: inference_model.attachToDevice() val_acc = test(inference_model, test_data, opts) inference_model.detachFromDevice() return val_acc return validation_func
def run_nn(dataset_name, target, n_splits, is_plot=False, batch_size=batch_size, epochs=epochs): # Данные ns, X, Y = dt.get_data(target=target, dataset_name=dataset_name) # Обучение kfold = StratifiedKFold(n_splits=n_splits, shuffle=True) cvscores = [] non_cat_title, cat_title = get_titles_lists(dataset_name) it = 0 for train, test in kfold.split(X, Y): scaler = preprocessing.StandardScaler() x_train, y_train = get_x_y_data(X, Y, train, scaler, non_cat_title, cat_title) x_test, y_test = get_x_y_data(X, Y, test, scaler, non_cat_title, cat_title) # Модель model = md.get_simple_nn(X.shape[1]) history = model.fit(x_train.values, y_train, epochs=epochs, batch_size=batch_size, verbose=0) # if is_plot: # plot(history) scores = model.evaluate(x_test.values, y_test, verbose=0, batch_size=batch_size) print("%s: %.2f" % (model.metrics_names[1], scores[1])) cvscores.append(scores[1]) predicted = model.predict(x_test.values, batch_size=1) print("corr: {:.2f}".format( np.corrcoef(y_test, [i[0] for i in predicted])[0][1])) save_histogram(predicted, y_test, it) x_test_sc = pd.DataFrame(scaler.inverse_transform( x_test[non_cat_title]), columns=non_cat_title, index=x_test.index) x_test_sc[target] = y_test x_test_sc[target + u'(сеть)'] = predicted x_test_sc.combine_first( pd.DataFrame(ns)).dropna().to_excel(dp.results['NN'] + '/res{}.xlsx'.format(str(it))) it += 1 print("%.2f (+/- %.2f)" % (np.mean(cvscores), np.std(cvscores))) return [np.mean(cvscores), np.std(cvscores)]
import datasets import math from wrappers.correlation import ace import numpy as np from collections import defaultdict sc = defaultdict(int) total = defaultdict(int) for ds in datasets.get_datasets(): print ds['name'] Xc, yc = datasets.get_data(ds, convert='numbers', standardize=False) cat_cor = ace(Xc, yc, range(Xc.shape[1]), ds['rtype']) num_cor = ace(Xc, yc, [], ds['rtype']) for b in (0.4,0.5,0.6,0.7,0.8,0.85,0.9,0.95,1.0,1.1,1.2,1.3): for i,c in enumerate(datasets.get_columns(ds)): bias = len(np.unique(Xc[:,i]))*1.0/(b*Xc.shape[0]) guess = 'num' if num_cor[i]>cat_cor[i]-bias else 'cat' if guess == 'cat' and i in datasets.get_column_index(ds) or guess == 'num' and i not in datasets.get_column_index(ds): sc[b] += 1 total[b] += 1 #print '%-30s %10.7f %10.7f %10.7f %s' % (c,num_cor[i],cat_cor[i],bias,guess) print '%6.2f %6.2f%%' % (b,sc[b]*100/total[b])
### Some constants ### data_file = './data/stock.csv' sequence_len = 12 test_size = 0.3 epochs = 15 learning_rate = 5e-2 ss = MinMaxScaler(feature_range=(-1, 1)) model = Network(rnn_cell='lstm', sigmoid=False, rnn_units=2) optimizer = optim.Adam(model.parameters(), lr=learning_rate) criterion = nn.MSELoss() cosine_annealing = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=learning_rate) X_train, X_test = get_data(data_file, sequence_len=sequence_len, test_size=test_size) X_train = ss.fit_transform(X_train) X_test = ss.transform(X_test) X_train = torch.Tensor(X_train).type(torch.FloatTensor) X_test = torch.Tensor(X_test).type(torch.FloatTensor) def _make_sequence_prediction(model, inputs): outputs = [] print('[INFO] Predicting ... ') with tqdm(total=len(inputs), file=sys.stdout, colour='green') as pbar: for i, input_ in enumerate(inputs): if (i == 0):
import datasets dataset="pugeault" (x_train, y_train), (x_test, y_test), input_shape,num_classes,labels= datasets.get_data(dataset) print(f"Images shape {input_shape}") print(f"classes {num_classes}, labels:\n {labels}") print(f"Train samples: {y_train.shape[0]}, Test samples: {y_test.shape[0]}") import matplotlib.pyplot as plt print(x_train.shape) initial_sample=0 samples=64 skip= y_train.shape[0] // samples grid_cols=8 grid_rows=samples // grid_cols if samples % grid_cols >0: grid_rows+=1 f,axes=plt.subplots(grid_rows,grid_cols) for axr in axes: for ax in axr: ax.get_xaxis().set_visible(False) ax.get_yaxis().set_visible(False) for i in range(samples): i_sample=i*skip+initial_sample klass = y_train[i_sample].argmax() row=i // grid_cols col=i % grid_cols
def advs_train(dataset='cifar-10', loss_name='ce', epochs=120, dynamic_epoch=100, batch_size=128, fosc_max=0.5, epsilon=0.031): """ Adversarial training with PGD attack. """ print( 'DynamicAdvsTrain - Data set: %s, loss: %s, epochs: %s, dynamic_epoch: %s, batch: %s, epsilon: %s' % (dataset, loss_name, epochs, dynamic_epoch, batch_size, epsilon)) X_train, Y_train, X_test, Y_test = get_data(dataset, clip_min=0., clip_max=1., onehot=True) n_images = X_train.shape[0] image_shape = X_train.shape[1:] n_class = Y_train.shape[1] print("n_images:", n_images, "n_class:", n_class, "image_shape:", image_shape) model = get_model(dataset, input_shape=image_shape, n_class=n_class, softmax=True) # model.summary() # create loss if loss_name == 'ce': loss = cross_entropy else: print("New loss function should be defined first.") return optimizer = SGD(lr=0.01, decay=1e-4, momentum=0.9) model.compile(loss=loss, optimizer=optimizer, metrics=['accuracy']) # data augmentation if dataset in ['mnist']: datagen = ImageDataGenerator() elif dataset in ['cifar-10']: datagen = ImageDataGenerator(rotation_range=10, width_shift_range=0.2, height_shift_range=0.2, horizontal_flip=True) else: datagen = ImageDataGenerator(width_shift_range=0.2, height_shift_range=0.2, horizontal_flip=True) datagen.fit(X_train) # pgd attack for training attack = LinfPGDAttack(model, epsilon=epsilon, eps_iter=epsilon / 4, nb_iter=10, random_start=True, loss_func='xent', clip_min=np.min(X_train), clip_max=np.max(X_train)) # initialize logger mylogger = Logger(K.get_session(), model, X_train, Y_train, X_test, Y_test, dataset, loss_name, epochs, suffix='%s' % epsilon) batch_iterator = datagen.flow(X_train, Y_train, batch_size=batch_size) start_time = time.time() for ep in range(epochs): # learning rate decay if (ep + 1) == 60: lr = float(K.get_value(model.optimizer.lr)) K.set_value(model.optimizer.lr, lr / 10.0) if (ep + 1) == 100: lr = float(K.get_value(model.optimizer.lr)) K.set_value(model.optimizer.lr, lr / 10.0) lr = float(K.get_value(model.optimizer.lr)) # a simple linear decreasing of fosc fosc = fosc_max - fosc_max * (ep * 1.0 / dynamic_epoch) fosc = np.max([fosc, 0.0]) steps_per_epoch = int(X_train.shape[0] / batch_size) pbar = tqdm(range(steps_per_epoch)) for it in pbar: batch_x, batch_y = batch_iterator.next() batch_advs, fosc_batch = attack.perturb(K.get_session(), batch_x, batch_y, batch_size, ep, fosc) probs = model.predict(batch_advs) loss_weight = np.max(-batch_y * np.log(probs + 1e-12), axis=1) if it == 0: fosc_all = fosc_batch else: fosc_all = np.concatenate((fosc_all, fosc_batch), axis=0) if ep == 0: loss, acc = model.train_on_batch(batch_advs, batch_y) else: loss, acc = model.train_on_batch(batch_advs, batch_y, sample_weight=loss_weight) pbar.set_postfix(acc='%.4f' % acc, loss='%.4f' % loss) print('All time:', time.time() - start_time) log_path = './log' file_name = os.path.join( log_path, 'BatchSize_{}_Epoch_{}_fosc.npy'.format(batch_size, ep)) np.save(file_name, fosc_all) val_loss, val_acc = model.evaluate(X_test, Y_test, batch_size=batch_size, verbose=0) logs = { 'acc': acc, 'loss': loss, 'val_acc': val_acc, 'val_loss': val_loss } print( "Epoch %s - loss: %.4f - acc: %.4f - val_loss: %.4f - val_acc: %.4f" % (ep, loss, acc, val_loss, val_acc)) # save the log and model every epoch mylogger.on_epoch_end(epoch=ep, logs=logs) model.save_weights("model/advs_%s_%s_%s_%s.hdf5" % (dataset, loss_name, epsilon, ep))
def main(): dataset = 'mnist' batch_size = 100 n_classes = 10 learning_rate = 1e-3 name = dataset oldmodel = None inits = { 'weights_init': IsotropicGaussian(0.01), 'biases_init': Constant(0.), } # ----------------------------------------------------------------------- image_size, channels, data_train, data_valid, data_test = \ datasets.get_data(dataset) img_height, img_width = image_size train_stream = Flatten(DataStream.default_stream( data_train, iteration_scheme=SequentialScheme( data_train.num_examples, batch_size))) test_stream = Flatten(DataStream.default_stream( data_test, iteration_scheme=SequentialScheme( data_test.num_examples, batch_size))) logging.info("experiment dataset: %s" % dataset) # --------------- Building Model ---------------- glim_net = GlimpseNetwork(dim=128, n_channels=channels, img_height=img_height, img_width=img_width, N=7, name='glimpse_net', **inits) # output (n) core = CoreNetwork(input_dim=128, dim=256, name='core_net', **inits) loc_net = LocationNetwork(input_dim=256, loc_emb=2, std=0.18, non_hetro=False, name='loc_net', **inits) action = ActionNetwork(input_dim=256, n_classes=n_classes, name='action', **inits) ram = RAM(core=core, glimpes_network=glim_net, location_network=loc_net, action_network=action, n_steps=8, name='RAM', random_init_loc=False, **inits) ram.initialize() # ------------------------------------------------------------- img = T.matrix('features') y = T.imatrix('targets') y = y.flatten() loc_sample, actions, _, _, loc_u = ram.out(img) loc_sample = loc_sample[:-1] loc_u = loc_u[:-1] # classification task actions = T.repeat(actions[1:][-1][None, :, :], loc_sample.shape[0], axis=0) # get loc network param # ---------------- Building Reinforce alforithm ---------------- loc_bricks = list(loc_net.children) # + list(glim_net.children) others_bricks = list(core.children) + list(action.children) + \ list(glim_net.children) reinforce = REINFORCE() cost_re, reward, baseline = \ reinforce.build_reinforce_cost_reward_base(y, actions) cg_rein = ComputationGraph([cost_re]) loc_params = VariableFilter(roles=[PARAMETER], bricks=loc_bricks)(cg_rein.variables) loc_grad = reinforce.build_reinforce_grad(cost_re, loc_u, loc_sample, loc_params, reward, baseline, loc_net.std) y_dis = actions[-1] cost_true = CategoricalCrossEntropy().apply(y, y_dis) # cost_true = cost_re cg = ComputationGraph([cost_true]) # filter out initial_state others_params = VariableFilter(roles=[PARAMETER], bricks=others_bricks)(cg.variables) other_grad = T.grad(cost_true, others_params) # Hybrid Cost all_grad = loc_grad + other_grad all_params = loc_params + others_params gradients = {param: grad for param, grad in zip(all_params, all_grad)} algorithm = GradientDescent( cost=cost_true, gradients=gradients, parameters=all_params, step_rule=CompositeRule([ StepClipping(10.), Adam(learning_rate)]), ) algorithm.add_updates(cg_rein.updates) # ------------------------------------------------------------------------ # Setup monitors cost_true.name = 'cost_true' cost_re.name = 'reinforce cost' # avg_action = actions.mean(1) # avg_action.name = 'avg_action' acc = T.cast(T.eq(y, T.argmax(y_dis, axis=1)), 'float32').mean() acc.name = 'accuratcy' monitors = [cost_re, acc, cost_true] train_monitors = monitors monitor_img = data_train.data_sources[0][:10*20].reshape((200, -1)) # Live plotting... # plot_channels = [ # ["cost_true", "reinforce_cost"] # ] # ------------------------------------------------------------ # plotting_extensions = [ # Plot(name, channels=plot_channels) # ] subdir = './exp/' + name + "-" + time.strftime("%Y%m%d-%H%M%S") if not os.path.exists(subdir): os.makedirs(subdir) main_loop = MainLoop( model=Model(cost_true), data_stream=train_stream, algorithm=algorithm, extensions=[ Timing(), EarlyStopping('test_cost_true', iterations=100), FinishAfter(after_n_epochs=500), TrainingDataMonitoring( train_monitors, prefix="train", after_epoch=True), DataStreamMonitoring( monitors, test_stream, updates=cg_rein.updates, prefix="test"), ProgressBar(), Printing(), SampleAttentionCheckpoint(monitor_img=monitor_img, image_size=image_size[0], channels=channels, save_subdir='{}'.format(subdir), before_training=True, after_epoch=True), PartsOnlyCheckpoint("{}/{}".format(subdir, name), before_training=True, after_epoch=True, save_separately=['log', 'model'])]) if oldmodel is not None: print("Initializing parameters with old model %s" % oldmodel) with open(oldmodel, "rb") as f: oldmodel = pickle.load(f) main_loop.model.set_parameter_values( oldmodel.get_parameter_values()) del oldmodel main_loop.run()
import sklearn.model_selection import sklearn.metrics import pandas as pd import numpy as np # Import files from source directory import os import sys module_path = os.path.abspath(os.path.join('../src/')) if module_path not in sys.path: sys.path.append(module_path) import hyperplane, datasets from model import BATDepth PROCESSED_DATASETS = { # 'wpbc': datasets.get_data("../data/breast-cancer/wpbc.data", 'wpbc'), # 'wdbc': datasets.get_data("../data/breast-cancer/wdbc.data", 'wdbc'), # 'breast_cancer_yugoslavia': datasets.get_data("../data/breast-cancer/breast-cancer.data", 'breast_cancer_yugoslavia'), 'breast_cancer_wisconsin': datasets.get_data("../data/breast-cancer/breast-cancer-wisconsin.data", 'breast_cancer_wisconsin'), } model, train_data, test_data = datasets.get_model( PROCESSED_DATASETS['breast_cancer_wisconsin']) c_tree = BATDepth(model, log=False) output_decision_tree = c_tree.fit() test_predict = output_decision_tree.predict( test_data[test_data.columns.difference(['target'])].values) print('BATDepth accuracy: {}'.format( (test_predict == test_data['target']).sum() / len(test_predict)))
import numpy as np from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier import datasets from sklearn.cross_validation import cross_val_score, KFold from sklearn.metrics import make_scorer, mean_absolute_error from wrappers.correlation import ace,rcorrs,rcorrp, bcdcor from collections import defaultdict MAD = make_scorer(mean_absolute_error, greater_is_better=False) sc=defaultdict(lambda:defaultdict(lambda:defaultdict(int))) for ds in datasets.get_datasets(): Xc, yc = datasets.get_data(ds, convert='numbers', standardize=False) kf = KFold(Xc.shape[0], 5, shuffle=True) for vi in (bcdcor, rcorrp, rcorrs, ace): cor = vi(Xc, yc, datasets.get_column_index(ds)) #print zip(ds['category']+ds['numeric'],cor) for drop in range(10): vi_name = vi.__name__ X, y = datasets.get_data(ds, drop=np.argsort(cor)[:drop]) if ds['rtype'] == 'Binary': metric = 'log_loss' clf = RandomForestClassifier(n_estimators=200,min_samples_leaf=5,n_jobs=1) else: metric = MAD clf = RandomForestRegressor(n_estimators=200,min_samples_leaf=5,n_jobs=1) sc[ds['name']][vi_name][drop] = cross_val_score(clf, X, y, cv=kf, n_jobs=3, scoring=metric).mean() print ds['name']+' '+vi_name+" "+str(drop)+' '+str(sc[ds['name']][vi_name][drop])
if len(sys.argv) > 1 and sys.argv[1] == 'r': resume = True hidden_number = 256 learning_rate = 0.01 bs = 128 saving_path = 'models' start_epoch = 0 end_epoch = 200 os.environ["CUDA_VISIBLE_DEVICES"] = "0,1" cuda_available = torch.cuda.is_available() #cuda_available = False para_padding_a, word2ix, ix2word = ds.get_data() para_padding_t = torch.from_numpy(para_padding_a) net = nets.PoetryModel(hidden_number, len(word2ix)) if resume is True: checkpoints = torch.load(os.path.join(saving_path, 'best_model.t7')) net.load_state_dict(checkpoints['net']) start_epoch = checkpoints['epoch'] net = net.cuda() if cuda_available is True else net dataloader = DataLoader(para_padding_t, batch_size = bs, shuffle = True, num_workers = 1) optimizer = torch.optim.Adam(net.parameters(), lr = learning_rate) criterion = nn.CrossEntropyLoss()
if opts.warmup_epoch > 0: lr_scheduler = WarmUpLRDecorator(optimizer=optimizer, lr_scheduler=lr_scheduler, warmup_epoch=opts.warmup_epoch) return lr_scheduler if __name__ == '__main__': run_opts = parse_arguments() logging.info("Loading the data") model_opts = create_model_opts(run_opts) train_data = datasets.get_data(run_opts, model_opts, train=True, async_dataloader=True) logging.info("Initialize the model") model = models.get_model(run_opts, datasets.datasets_info[run_opts.data], pretrained=False) model.train() optimizer = get_optimizer(run_opts, model) lr_scheduler = get_lr_scheduler(run_opts, optimizer) training_model = convert_to_ipu_model(model, run_opts, optimizer) training_validation_func = get_validation_function( run_opts, model) if run_opts.validation_mode == "during" else None train(training_model, train_data, run_opts, lr_scheduler, range(1, run_opts.epoch + 1), optimizer, training_validation_func)
if act[i]==0: sc += 2 * pred[i] else: if pred[i]<=0: sc += 2 * (act[i] * math.log(act[i]/0.1) - (act[i] - pred[i])) else: sc += 2 * (act[i] * math.log(act[i]/pred[i]) - (act[i] - pred[i])) return sc/act.shape[0] RMSLE = make_scorer(rmsle, greater_is_better=False) PSDEV = make_scorer(psdev, greater_is_better=False) #for ds in datasets.get_datasets(rtype='Positive'): #,name='census_1990_small'): for ds in datasets.get_datasets(name='census_2012h_small'): #if 'census' in ds['name']: continue X, y = datasets.get_data(ds,standardize=False) kf = KFold(X.shape[0], 2, shuffle=True, random_state=1234) for a in (0.1, 0.3, 0.6, 0.9): for lm in (1, 0.1, 0.01): clf1 = CGLM(distribution='Poisson',trace=False) clf2 = Ridge(alpha=lm) clf3 = ElasticNetC(distribution='Poisson', alpha=lm, l1_ratio=a, tolerance=0.001) #clf3t = ElasticNetC(distribution='Tweedie', alpha=lm, l1_ratio=a, tolerance=0.001,p=1.5) #clf3g = ElasticNetC(distribution='Gamma', alpha=lm, l1_ratio=a, tolerance=0.001) clf4 = GlmnetWrapper(**{'family': 'gaussian', 'alpha': a, 'lambda': lm, 'maxit': 300}) #clf4 = GlmnetWrapper(**{'family': 'poisson', 'alpha': a, 'lambda': lm, 'maxit': 300}) score_func = PSDEV #'roc_auc' for clf in (clf1, clf2, clf3, clf4): #for clf in (clf1, clf2, clf3, clf3t, clf3g, clf4): if clf != clf1 or (a == 0.1 and lm == 10): cn=clf.__class__.__name__
import sklearn.ensemble import sklearn.model_selection import sklearn.metrics import pandas as pd import numpy as np # Import files from source directory import os import sys module_path = os.path.abspath(os.path.join('../src/')) if module_path not in sys.path: sys.path.append(module_path) import hyperplane, datasets from model import BATDepth PROCESSED_DATASETS = { 'htru': datasets.get_data("../data/htru/HTRU_2.csv", 'htru'), } model, train_data, test_data = datasets.get_model(PROCESSED_DATASETS['htru'], n_estimators = 2) c_tree = BATDepth(model, log = False) output_decision_tree = c_tree.fit() test_predict = output_decision_tree.predict(test_data[test_data.columns.difference(['target'])].values) print('BATDepth accuracy: {}'.format((test_predict == test_data['target']).sum()/len(test_predict)))
y.backward(torch.ones_like(y)) jacob = x.grad.detach() return jacob, target.detach(), y.detach(), out.detach() device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") savedataset = args.dataset dataset = 'fake' if 'fake' in args.dataset else args.dataset args.dataset = args.dataset.replace('fake', '') if args.dataset == 'cifar10': args.dataset = args.dataset + '-valid' searchspace = nasspace.get_search_space(args) if 'valid' in args.dataset: args.dataset = args.dataset.replace('-valid', '') train_loader = datasets.get_data(args.dataset, args.data_loc, args.trainval, args.batch_size, args.augtype, args.repeat, args) os.makedirs(args.save_loc, exist_ok=True) filename = f'{args.save_loc}/{args.save_string}_{args.score}_{args.nasspace}_{savedataset}{"_" + args.init + "_" if args.init != "" else args.init}_{"_dropout" if args.dropout else ""}_{args.augtype}_{args.sigma}_{args.repeat}_{args.trainval}_{args.batch_size}_{args.maxofn}_{args.seed}' accfilename = f'{args.save_loc}/{args.save_string}_accs_{args.nasspace}_{savedataset}_{args.trainval}' if args.dataset == 'cifar10': acc_type = 'ori-test' val_acc_type = 'x-valid' else: acc_type = 'x-test' val_acc_type = 'x-valid' scores = np.zeros(len(searchspace)) try:
import data_paths as dp import titles as tl import datasets as dt from matplotlib import pyplot as plt from sklearn.model_selection import StratifiedKFold from sklearn import preprocessing from sklearn import svm from sklearn import neighbors # Параметры обучения C = 10.0 n_trees = 256 # Данные target = u'прочность' ns, X, Y = dt.get_data(target=target) # Обучение kfold = StratifiedKFold(n_splits=10, shuffle=True) cvscores = [] it = 0 for train, test in kfold.split(X, Y): scaler = preprocessing.StandardScaler() sc_data = X.loc[train, tl.full_hard_non_cat_title] sc_data = pd.DataFrame(scaler.fit_transform(sc_data), index=sc_data.index, columns=sc_data.columns) ct_data = X.loc[train, tl.full_hard_cat_title] x_train = pd.concat([sc_data, ct_data], axis=1)
def get_numeric_features(): X, y = get_data('../data/trainDF.csv') dtypes = pd.DataFrame(X.dtypes.rename('type')).reset_index().astype('str') numeric = dtypes[(dtypes.type.isin(['int64', 'float64']))]['index'].values return numeric
) args.normalization_location = 'ipu' utils.handle_distributed_settings(args) return args if __name__ == '__main__': args = parse_arguments() utils.Logger.setup_logging_folder(args) if args.use_popdist: opts = popdist.poptorch.Options( ipus_per_replica=len(args.pipeline_splits) + 1) else: opts = poptorch.Options() opts.replicationFactor(args.replicas) opts.deviceIterations(args.device_iterations) dataloader = datasets.get_data(args, opts, train=False, async_dataloader=False) model = models.get_model(args, datasets.datasets_info[args.data], pretrained=not args.random_weights) model.eval() opts = utils.inference_settings(args, copy.deepcopy(opts)) inference_model = poptorch.inferenceModel(model, opts) benchmark(inference_model, dataloader, args)
from sklearn.preprocessing import StandardScaler import pickle import os model_name = 'OneClassSVM' #nu = 0.0005 kernel = 'rbf' #gamma = 0.0002 #training path model_home = os.getcwd() normal_path = model_home + '/datasets/normal2.xlsx' abnormal_path = model_home + '/datasets/abnormal.xlsx' print(abnormal_path) X_train, X_val, val_set, test_set = datasets.get_data(model_name, normal_path, abnormal_path) sc = StandardScaler() sc.fit(X_train) X_train_std = sc.transform(X_train) def fn(args): nu, gamma = args params = {'nu': nu, 'kernel': kernel, 'gamma': gamma} oneclasssvm = creat_model(model_name, params) oneclasssvm.fit(X_train_std) errors = 0 for data in val_set: X = data.X y = data.y
help='learning rate of optimizer') parser.add_argument('--seed', default=45, type=int) # LTEC hyperparameters parser.add_argument('-m', '--num_of_prediction', default=5, type=int) parser.add_argument('-w', '--warming_up_period', default=10, type=int) config = parser.parse_args() os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" os.environ["CUDA_VISIBLE_DEVICES"] = '%d' % config.gpu # load dataset train_x, train_y, test_x, test_y, clean_y = get_data( dataset=config.dataset, noise_ratio=config.noise_ratio, noise_type=config.noise_type, random_shuffle=True, seed=config.seed) np.random.seed(config.seed) n_samples = train_x.shape[0] x_shape = train_x.shape[1:] y_shape = train_y.shape[1] n_batches = n_samples // batch_size + 1 # define model model = convnet(config, x_shape=x_shape, y_shape=y_shape, mom2=0.1) forget_rate = config.noise_ratio / 100. remember_rate = 1. - forget_rate
import datasets from wrappers.elastic_net_c import ElasticNetC import time from wrappers.glmnet import GlmnetWrapper from wrappers.zeroinfl import ZeroInflWrapper from wrappers.glm_c import CGLM from sklearn.cross_validation import cross_val_score, KFold from sklearn.metrics import make_scorer, mean_absolute_error from sklearn.linear_model import SGDRegressor, Ridge, LogisticRegression, SGDClassifier, ElasticNet MAD = make_scorer(mean_absolute_error, greater_is_better=False) score_func='r2' for ds in datasets.get_datasets(name='mets_short'): X, y = datasets.get_data(ds) kf = KFold(X.shape[0], 15, shuffle=True, random_state=1234) for a in (0.1, 0.5, 1.0, 1.5): clf1 = Ridge(alpha=a) clf2 = CGLM(distribution='Poisson',trace=False) clf3 = ZeroInflWrapper() for clf in (clf1,clf2,clf3): if a!=0.1 and clf!=clf1: continue st = time.time() print '%-20s %20s a %3.1f sc %9.4f tm %5.2f' % \ (ds['name'], clf.__class__.__name__, a, cross_val_score(clf, X, y, cv=kf, scoring=score_func, n_jobs=3).mean(), time.time()-st)
def feature_visualization(model_name='ce', dataset='mnist', num_classes=10, noise_ratio=40, n_samples=100): """ This is to show how features of incorretly labeled images are overffited to the wrong class. plot t-SNE 2D-projected deep features (right before logits). This will generate 3 plots in a grid (3x1). The first shows the raw features projections of two classes of images (clean label + noisy label) The second shows the deep features learned by cross-entropy after training. The third shows the deep features learned using a new loss after training. :param model_name: a new model other than crossentropy(ce), can be: boot_hard, boot_soft, forward, backward, lid :param dataset: :param num_classes: :param noise_type; :param noise_ratio: :param epochs: to find the last epoch :param n_samples: :return: """ print('Dataset: %s, model_name: ce/%s, noise ratio: %s%%' % (model_name, dataset, noise_ratio)) features_ce = np.array([None, None]) features_other = np.array([None, None]) # # load pre-saved to avoid recomputing # feature_tmp = "lof/representation_%s_%s.npy" % (dataset, noise_ratio) # if os.path.isfile(feature_tmp): # data = np.load(feature_tmp) # features_input = data[0] # features_ce = data[1] # features_other = data[2] # # plot(model_name, dataset, noise_ratio, features_input, features_ce, features_other) # return # load data X_train, Y_train, X_test, Y_test = get_data(dataset) Y_noisy = np.load("data/noisy_label_%s_%s.npy" % (dataset, noise_ratio)) Y_noisy = Y_noisy.reshape(-1) # sample training set cls_a = 0 cls_b = 3 # find smaples labeled to class A and B cls_a_idx = np.where(Y_noisy == cls_a)[0] cls_b_idx = np.where(Y_noisy == cls_b)[0] # sampling for efficiency purpose cls_a_idx = np.random.choice(cls_a_idx, n_samples, replace=False) cls_b_idx = np.random.choice(cls_b_idx, n_samples, replace=False) X_a = X_train[cls_a_idx] X_b = X_train[cls_b_idx] image_shape = X_train.shape[1:] model = get_model(dataset, input_tensor=None, input_shape=image_shape) sgd = SGD(lr=0.01, momentum=0.9) #### get deep representations of ce model model_path = 'model/ce_%s_%s.hdf5' % (dataset, noise_ratio) model.load_weights(model_path) model.compile(loss=cross_entropy, optimizer=sgd, metrics=['accuracy']) rep_a = get_deep_representations(model, X_a, batch_size=100).reshape( (X_a.shape[0], -1)) rep_b = get_deep_representations(model, X_b, batch_size=100).reshape( (X_b.shape[0], -1)) rep_a = TSNE(n_components=2).fit_transform(rep_a) rep_b = TSNE(n_components=2).fit_transform(rep_b) features_ce[0] = rep_a features_ce[1] = rep_b #### get deep representations of other model model_path = 'model/%s_%s_%s.hdf5' % (model_name, dataset, noise_ratio) model.load_weights(model_path) model.compile(loss=cross_entropy, optimizer=sgd, metrics=['accuracy']) rep_a = get_deep_representations(model, X_a, batch_size=100).reshape( (X_a.shape[0], -1)) rep_b = get_deep_representations(model, X_b, batch_size=100).reshape( (X_b.shape[0], -1)) rep_a = TSNE(n_components=2).fit_transform(rep_a) rep_b = TSNE(n_components=2).fit_transform(rep_b) features_other[0] = rep_a features_other[1] = rep_b # plot fig = plt.figure(figsize=(12, 5)) gs = gridspec.GridSpec(1, 2, wspace=0.15) a_clean_idx = Y_train[cls_a_idx] == Y_noisy[cls_a_idx] a_noisy_idx = Y_train[cls_a_idx] != Y_noisy[cls_a_idx] b_clean_idx = Y_train[cls_b_idx] == Y_noisy[cls_b_idx] b_noisy_idx = Y_train[cls_b_idx] != Y_noisy[cls_b_idx] ## plot features learned by cross-entropy ax = fig.add_subplot(gs[0, 0]) A = features_ce[0] B = features_ce[1] # clean labeld class A samples plot ax.scatter(A[a_clean_idx][:, 0].ravel(), A[a_clean_idx][:, 1].ravel(), c='b', marker='o', s=10, label='class A: clean') ax.scatter(A[a_noisy_idx][:, 0].ravel(), A[a_noisy_idx][:, 1].ravel(), c='m', marker='x', s=30, label='class A: noisy') ax.scatter(B[b_clean_idx][:, 0].ravel(), B[b_clean_idx][:, 1].ravel(), c='r', marker='o', s=10, label='class B: clean') ax.scatter(B[b_noisy_idx][:, 0].ravel(), B[b_noisy_idx][:, 1].ravel(), c='c', marker='x', s=30, label='class B: noisy') ax.set_title("cross-entropy", fontsize=15) legend = ax.legend(loc='lower center', ncol=2) plt.setp(legend.get_texts(), fontsize=15) ax = fig.add_subplot(gs[0, 1]) A = features_other[0] B = features_other[1] ax.scatter(A[a_clean_idx][:, 0].ravel(), A[a_clean_idx][:, 1].ravel(), c='b', marker='o', s=10, label='class A: clean') ax.scatter(A[a_noisy_idx][:, 0].ravel(), A[a_noisy_idx][:, 1].ravel(), c='m', marker='x', s=30, label='class A: noisy') ax.scatter(B[b_clean_idx][:, 0].ravel(), B[b_clean_idx][:, 1].ravel() - 5, c='r', marker='o', s=10, label='class B: clean') ax.scatter(B[b_noisy_idx][:, 0].ravel(), B[b_noisy_idx][:, 1].ravel(), c='c', marker='x', s=30, label='class B: noisy') ax.set_title("D2L", fontsize=15) legend = ax.legend(loc='lower center', ncol=2) plt.setp(legend.get_texts(), fontsize=15) fig.savefig("plots/representations_%s_%s_%s.png" % (model_name, dataset, noise_ratio), dpi=300) plt.show()
# epochs.append(d3 // 2) # epochs.append(d3 // 2) # curriculum_type = "hard" score_path = dataset + "_scores.p" os.environ["PYTHONHASHSEED"] = str(seed) random.seed(seed) np.random.seed(seed) tf.random.set_seed(seed) X_train, y_train, X_test, y_test = datasets.get_data( dataset) tf.keras.backend.clear_session() gc.collect() if dataset == 'bert_imdb1' or True: try: tpu = tf.distribute.cluster_resolver.TPUClusterResolver( ) tf.config.experimental_connect_to_cluster(tpu) tf.tpu.experimental.initialize_tpu_system(tpu) strategy = tf.distribute.experimental.TPUStrategy( tpu) # Create model with strategy.scope(): model = models.get_model(dataset)
import numpy as np import datasets def get_accuracy_metric(col): eda_report = {} rp = _get_report_level_one(eda_report, col, col.dtype) return rp['metric_options']['accuracy']['short_name'] #for ds in datasets.get_datasets(name=['fastiron-train-30k','kickcars_train_full','census_1990_small','french_damage_cost','allstate_nonzero_small','amazon_small_no-c','credit_full','bank_marketing_small','bio_grid_small','mets','trainingDataWithoutNegativeWeights','census_2012h_small']): #for ds in datasets.get_datasets(name='trainingDataWithoutNegativeWeights'): for ds in datasets.get_datasets(name='fastiron-train-30k'): #for ds in datasets.get_datasets(name='fastiron_small'): #for ds in datasets.get_datasets(name='kickcars_train_full'): #for ds in datasets.get_datasets(name='french_damage_cost'): #for ds in datasets.get_datasets(name='census_1990_small'): X, y = datasets.get_data(ds,standardize=False,convert='numbers') cols = datasets.get_columns(ds) cats = datasets.get_columns(ds,'category') sc1={} sc2={} klist=(0,) #klist = (1,5,10,20,40,80) df=pandas.DataFrame(np.hstack((X,np.reshape(y,(-1,1)))),columns=cols+[ds['target']]) rm = get_accuracy_metric(df[ds['target']]) mdir = direction_by_name(rm) mfunc = metric_by_name(rm) rnk1={} rnk2={} for K in klist: sys.stderr.write( ds['name']+' '+str(K)+'\n') if ds['rtype']=='Binary':
def train(dataset='mnist', model_name='d2l', batch_size=128, epochs=50, noise_ratio=0): """ Train one model with data augmentation: random padding+cropping and horizontal flip :param dataset: :param model_name: :param batch_size: :param epochs: :param noise_ratio: :return: """ print('Dataset: %s, model: %s, batch: %s, epochs: %s, noise ratio: %s%%' % (dataset, model_name, batch_size, epochs, noise_ratio)) # load data X_train, y_train, X_test, y_test = get_data(dataset, noise_ratio, random_shuffle=True) # X_train, y_train, X_val, y_val = validatation_split(X_train, y_train, split=0.1) n_images = X_train.shape[0] image_shape = X_train.shape[1:] num_classes = y_train.shape[1] print("n_images", n_images, "num_classes", num_classes, "image_shape:", image_shape) # load model model = get_model(dataset, input_tensor=None, input_shape=image_shape, num_classes=num_classes) # model.summary() optimizer = SGD(lr=0.01, decay=1e-4, momentum=0.9) # for backward, forward loss # suppose the model knows noise ratio P = uniform_noise_model_P(num_classes, noise_ratio/100.) # create loss if model_name == 'forward': P = uniform_noise_model_P(num_classes, noise_ratio / 100.) loss = forward(P) elif model_name == 'backward': P = uniform_noise_model_P(num_classes, noise_ratio / 100.) loss = backward(P) elif model_name == 'boot_hard': loss = boot_hard elif model_name == 'boot_soft': loss = boot_soft elif model_name == 'd2l': loss = lid_paced_loss() else: loss = cross_entropy # model model.compile( loss=loss, optimizer=optimizer, metrics=['accuracy'] ) ## do real-time updates using callbakcs callbacks = [] if model_name == 'd2l': init_epoch = D2L[dataset]['init_epoch'] epoch_win = D2L[dataset]['epoch_win'] d2l_learning = D2LCallback(model, X_train, y_train, dataset, noise_ratio, epochs=epochs, pace_type=model_name, init_epoch=init_epoch, epoch_win=epoch_win) callbacks.append(d2l_learning) cp_callback = ModelCheckpoint("model/%s_%s_%s.hdf5" % (model_name, dataset, noise_ratio), monitor='val_loss', verbose=0, save_best_only=False, save_weights_only=True, period=1) callbacks.append(cp_callback) else: cp_callback = ModelCheckpoint("model/%s_%s_%s.hdf5" % (model_name, dataset, noise_ratio), monitor='val_loss', verbose=0, save_best_only=False, save_weights_only=True, period=epochs) callbacks.append(cp_callback) # tensorboard callback callbacks.append(TensorBoard(log_dir='./log/log')) # learning rate scheduler if use sgd lr_scheduler = get_lr_scheduler(dataset) callbacks.append(lr_scheduler) # acc, loss, lid log_callback = LoggerCallback(model, X_train, y_train, X_test, y_test, dataset, model_name, noise_ratio, epochs) callbacks.append(log_callback) # data augmentation if dataset in ['mnist', 'svhn']: datagen = ImageDataGenerator() elif dataset in ['cifar-10', 'cifar-100']: datagen = ImageDataGenerator( width_shift_range=0.2, height_shift_range=0.2, horizontal_flip=True) else: datagen = ImageDataGenerator( width_shift_range=0.1, height_shift_range=0.1, horizontal_flip=True) datagen.fit(X_train) # train model model.fit_generator(datagen.flow(X_train, y_train, batch_size=batch_size), steps_per_epoch=len(X_train) / batch_size, epochs=epochs, validation_data=(X_test, y_test), verbose=1, callbacks=callbacks )
""" Transform the mnist dataset into a bigger image, to test the effect of attention. """ from fuel.datasets.hdf5 import H5PYDataset import h5py import numpy as np import datasets image_size, channels, data_train, data_valid, data_test = \ datasets.get_data('mnist') data_sources = np.vstack([data_train.data_sources[0], data_test.data_sources[0], data_valid.data_sources[0]]) data_targets = np.vstack([data_train.data_sources[1], data_test.data_sources[1], data_valid.data_sources[1]]) batch_size = data_sources.shape[0] data_buffer = np.zeros((batch_size, 1, 60, 60)) # create the index start = np.random.random_integers( low=0, high=60 - 28, size=(batch_size, 2)) + 28 // 2 img_idx = np.arange(60 * 60).reshape(60, 60) for i, j in zip(range(batch_size), start): x, y = j.tolist() data_buffer[i, :, x-14: x+14, y-14: y+14] = \
def train(dataset='mnist', model_name='sl', batch_size=128, epochs=50, noise_ratio=0, asym=False, alpha=1.0, beta=1.0): """ Train one model with data augmentation: random padding+cropping and horizontal flip :param dataset: :param model_name: :param batch_size: :param epochs: :param noise_ratio: :return: """ print( 'Dataset: %s, model: %s, batch: %s, epochs: %s, noise ratio: %s%%, asymmetric: %s, alpha: %s, beta: %s' % (dataset, model_name, batch_size, epochs, noise_ratio, asym, alpha, beta)) # load data X_train, y_train, y_train_clean, X_test, y_test = get_data( dataset, noise_ratio, asym=asym, random_shuffle=False) n_images = X_train.shape[0] image_shape = X_train.shape[1:] num_classes = y_train.shape[1] print("n_images", n_images, "num_classes", num_classes, "image_shape:", image_shape) # load model model = get_model(dataset, input_tensor=None, input_shape=image_shape, num_classes=num_classes) # model.summary() if dataset == 'cifar-100': optimizer = SGD(lr=0.1, decay=5e-3, momentum=0.9) else: optimizer = SGD(lr=0.1, decay=1e-4, momentum=0.9) # create loss if model_name == 'ce': loss = cross_entropy elif model_name == 'sl': loss = symmetric_cross_entropy(alpha, beta) elif model_name == 'lsr': loss = lsr elif model_name == 'joint': loss = joint_optimization_loss elif model_name == 'gce': loss = generalized_cross_entropy elif model_name == 'boot_hard': loss = boot_hard elif model_name == 'boot_soft': loss = boot_soft elif model_name == 'forward': loss = forward(P) elif model_name == 'backward': loss = backward(P) else: print("Model %s is unimplemented!" % model_name) exit(0) # model model.compile(loss=loss, optimizer=optimizer, metrics=['accuracy']) if asym: model_save_file = "model/asym_%s_%s_%s.{epoch:02d}.hdf5" % ( model_name, dataset, noise_ratio) else: model_save_file = "model/%s_%s_%s.{epoch:02d}.hdf5" % ( model_name, dataset, noise_ratio) ## do real-time updates using callbakcs callbacks = [] if model_name == 'sl': cp_callback = ModelCheckpoint(model_save_file, monitor='val_loss', verbose=0, save_best_only=False, save_weights_only=True, period=1) callbacks.append(cp_callback) else: cp_callback = ModelCheckpoint(model_save_file, monitor='val_loss', verbose=0, save_best_only=False, save_weights_only=True, period=1) callbacks.append(cp_callback) # learning rate scheduler if use sgd lr_scheduler = get_lr_scheduler(dataset) callbacks.append(lr_scheduler) callbacks.append(SGDLearningRateTracker(model)) # acc, loss, lid log_callback = LoggerCallback(model, X_train, y_train, y_train_clean, X_test, y_test, dataset, model_name, noise_ratio, asym, epochs, alpha, beta) callbacks.append(log_callback) # data augmentation if dataset in ['mnist', 'svhn']: datagen = ImageDataGenerator() elif dataset in ['cifar-10']: datagen = ImageDataGenerator(width_shift_range=0.2, height_shift_range=0.2, horizontal_flip=True) else: datagen = ImageDataGenerator(rotation_range=20, width_shift_range=0.2, height_shift_range=0.2, horizontal_flip=True) datagen.fit(X_train) # train model model.fit_generator(datagen.flow(X_train, y_train, batch_size=batch_size), steps_per_epoch=len(X_train) / batch_size, epochs=epochs, validation_data=(X_test, y_test), verbose=1, callbacks=callbacks)
def train(epoch=10, batch_size=128, log_img=True): data, batch_count = [], 0 if dataset == 'cifar10' or dataset == 'mnist' or dataset == 'cifar100' or dataset == 'o-faces': data, batch_count = datasets.get_data( dataset, (input_width, input_height), (output_width, output_height), color_channels, batch_size) else: if dataset == 'celebA': data, batch_count = datasets.get_data( dataset, (input_width, input_height), (output_width, output_height), batch_size, color_channels) elif dataset == 'letters': data, batch_count = datasets.get_data( dataset, (input_width, input_height), (output_width, output_height), batch_size, color_channels, file_ext='*.png') for i in range(epoch): if (log_img): print('Saved Epoch', i) generator.save_weights(dataset + '_gen.h5') discriminator.save_weights(dataset + '_dis.h5') plot_output(dataset, i) for j in tqdm(range(batch_count)): # Input for the generator noise_input = np.random.rand(batch_size, noise_size) if dataset == 'cifar10' or dataset == 'mnist' or dataset == 'o-faces' or dataset == 'cifar100': image_batch = data[np.random.randint(0, data.shape[0], size=batch_size)] else: batch_files = data[j * batch_size:(j + 1) * batch_size] batch = [ scipy.misc.imread(batch_file).astype(np.float) for batch_file in batch_files ] image_batch = np.array(batch).astype(np.float32) image_batch = (image_batch - 127.5) / 127.5 # these are the predicted images from the generator predictions = generator.predict(noise_input, batch_size=batch_size) #print(predictions.shape) #print(image_batch.shape) # the discriminator takes in the real images and the generated images X = np.concatenate([predictions, image_batch]) # labels for the discriminator y_discriminator = [0] * batch_size + [1] * batch_size # Let's train the discriminator discriminator.trainable = True discriminator.train_on_batch(X, y_discriminator) # Let's train the generator noise_input = np.random.rand(batch_size, noise_size) y_generator = [1] * batch_size discriminator.trainable = False gan.train_on_batch(noise_input, y_generator)
def train(dataset, alpha, beta, thr): num_cores = multiprocessing.cpu_count() alpha = alpha beta = beta thr = thr w_lim = 1 c = 5 batch_budget = 250 bnn = 0 al_method = 'bvssb' dynamic = 0 budget = 0 epochs_init = 4 epochs = 3 batch_size = 64 noise_ratio = 30 NUM_CLASSES = { 'mnist': 10, 'svhn': 10, 'cifar-10': 10, 'cifar-100': 100, 'celeb': 20 } dataset = dataset init_noise_ratio = 0 data_ratio = 1.666666 X_train, y_train, X_test, y_test, x_val, y_val, un_selected_index = get_data( dataset, init_noise_ratio, data_ratio, random_shuffle=False) image_shape = X_train.shape[1:] model = get_model(bnn, dataset, input_tensor=None, input_shape=image_shape, num_classes=NUM_CLASSES[dataset]) optimizer = SGD(lr=0.1, decay=0.002, momentum=0.9) model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy']) datagen = ImageDataGenerator( featurewise_center=False, # set input mean to 0 over the dataset samplewise_center=False, # set each sample mean to 0 featurewise_std_normalization= False, # divide inputs by std of the dataset samplewise_std_normalization=False, # divide each input by its std zca_whitening=False, # apply ZCA whitening rotation_range= 0, # randomly rotate images in the range (degrees, 0 to 180) width_shift_range= 0.1, # randomly shift images horizontally (fraction of total width) height_shift_range= 0.1, # randomly shift images vertically (fraction of total height) horizontal_flip=True, # randomly flip images vertical_flip=False, # randomly flip images ) datagen.fit(X_train) h_quality = model.fit(datagen.flow(X_train, y_train, batch_size=batch_size), steps_per_epoch=X_train.shape[0] // batch_size, epochs=epochs_init, validation_data=(X_test, y_test), workers=num_cores) model_quality = clone_model(model) model_quality.set_weights(model.get_weights()) (X_train, y_train), (X_test, y_test) = mnist.load_data() X_train = X_train.reshape(-1, 28, 28, 1) X_test = X_test.reshape(-1, 28, 28, 1) X_train = X_train / 255.0 X_test = X_test / 255.0 means = X_train.mean(axis=0) # std = np.std(X_train) X_train = (X_train - means) # / std X_test = (X_test - means) # / std # they are 2D originally in cifar y_train = y_train.ravel() y_test = y_test.ravel() X_train = X_train.astype('float32') X_test = X_test.astype('float32') y_train = np_utils.to_categorical(y_train, NUM_CLASSES[dataset]) y_test = np_utils.to_categorical(y_test, NUM_CLASSES[dataset]) epochs_training = epochs training_noise_level = noise_ratio loss = 0 used_budget = 0 total_clean_num = 0 n_classes = NUM_CLASSES[dataset] # training_steps indicate how many data used in each batch training_steps = 1000 statistics_list = [[] for _ in range(21)] steps = int(np.ceil(len(un_selected_index) / float(training_steps))) for i in np.arange(steps): print("chunk:", i) model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy']) if i == 0: sub_un_selected_list = un_selected_index[0:training_steps] elif i != steps - 1: sub_un_selected_list = un_selected_index[i * training_steps:(i + 1) * training_steps] else: sub_un_selected_list = un_selected_index[i * training_steps:] if i != 0: model_quality = clone_model(model) model_quality.set_weights(model.get_weights()) X_in_iteration = X_train[sub_un_selected_list] y_true_iteration = y_train[sub_un_selected_list] y_noisy_iteration, noisy_idx = inject_noise( dataset, y_train[sub_un_selected_list], training_noise_level) y_predict, predict_prob, y_predict_label_second = BNN_label_predict( X_in_iteration, model, n_classes) clean_list, clean_pred_list, noisy_list, FN, TN, TP, FP, maxProbTP, maxProbFP = select_clean_noisy( X_in_iteration, y_noisy_iteration, y_true_iteration, y_predict, y_predict_label_second, predict_prob, model) oracle_list = [] if batch_budget > 0: num_al = len(noisy_list) print("batch_budget:", batch_budget) batch_list = range(0, training_steps) n = NUM_CLASSES[dataset] spent_s = 0 spent_w = 0 inf_ind, inf = BNN_active_selection(predict_prob, noisy_list, al_method, num_al) loss = model_reliability(model, x_val, y_val) strong_list, weak_list, spent_s, spent_w, first_yes, weak_questions, mean_w_q, unique_pic, true_label_rank = label_decider( noisy_list, y_true_iteration, y_predict, predict_prob, n, inf_ind, inf, loss, batch_budget, spent_s, spent_w, w_lim, c, alpha, beta, thr) statistics_list[0].append(len(strong_list)) statistics_list[1].append(len(weak_list)) statistics_list[2].append(first_yes) statistics_list[3].append(spent_w) statistics_list[4].append(mean_w_q) statistics_list[5].append(unique_pic) statistics_list[6].append(num_al) print("spent_s:", spent_s, "spent_s/c:", spent_s / c) print("spent_w:", spent_w, "weak yes:", len(weak_list)) oracle_list = np.append(strong_list, weak_list) if len(weak_list) == 0: oracle_list = strong_list if len(strong_list) == 0: oracle_list = weak_list y_noisy_iteration[oracle_list] = y_true_iteration[oracle_list] training_list = np.append(clean_list, oracle_list) x_train_iteration = X_in_iteration[training_list] y_train_iteration = y_noisy_iteration[training_list] print("train data shape:", x_train_iteration.shape) h_training_epoch_quality = model.fit_generator( datagen.flow(x_train_iteration, y_train_iteration, batch_size=batch_size), steps_per_epoch=y_train_iteration.shape[0] // batch_size + 1, epochs=epochs_training, validation_data=(X_test, y_test)) h_quality = combine_result(h_quality, h_training_epoch_quality) if i != 0 and h_quality.history['val_accuracy'][ -epochs_training - 1] - np.min(h_quality.history[ 'val_accuracy'][-epochs_training:]) > 0.2: model = clone_model(model_quality) model.set_weights(model_quality.get_weights()) s_bnn = 0 val_accc = h_quality.history['val_accuracy'] accc = h_quality.history['accuracy'] val_losss = h_quality.history['val_loss'] losss = h_quality.history['loss'] for i in range(0, 10): s_bnn = s_bnn + val_accc[-epochs_training * i - 1] print(val_accc[-epochs_training * i - 1]) average_valacc = s_bnn / 10 print("Final Acc:", average_valacc) statistics_list[14].append(val_accc) statistics_list[15].append(accc) statistics_list[16].append(val_losss) statistics_list[17].append(losss) end = time.time() statistics_list[18].append(end - start) print("timing: ", end - start) #################################### save ######################################## with open( 'WSstat_alpha' + str(alpha) + '+beta' + str(beta) + '+threshold' + str(thr) + '.pickle', 'wb') as file_pi: pickle.dump(statistics_list, file_pi) return average_valacc
# In[6]: from sklearn.decomposition import PCA from sklearn.pipeline import Pipeline from sklearn.grid_search import GridSearchCV from sklearn.linear_model import LogisticRegression, LogisticRegressionCV # ## Pre-Processed Data # In[19]: from datasets import get_data X, X_0, Y = get_data() # ### Principle Components Analysis # In[21]: n_components = [3, 4, 5] pca = PCA() logistic = LogisticRegression() pipe = Pipeline(steps=[('pca', pca), ('logistic', logistic)]) estimator = GridSearchCV(pipe, dict( pca__n_components=n_components,