def grid_search(max_features_list, n_estimators_list): max_f1 = 0 max_oversample = None max_scale = None max_mf = None max_nest = None for oversample in [True, False]: for scale in [True, False]: print("Oversample: {}, Scale: {}".format(oversample, scale)) X_train, X_test, y_train, y_test = split_data( oversample=oversample, scale=scale) for mf in max_features_list: for nest in n_estimators_list: f1 = fit_model(X_train, X_test, y_train, y_test, max_features=mf, n_estimators=nest) if f1 > max_f1: print("NEW MAX!") max_f1 = f1 max_oversample = oversample max_scale = scale max_mf = mf max_nest = nest return max_f1, max_oversample, max_scale, max_mf, max_nest
def train_model(config_path: str): writer = SummaryWriter() config = read_training_pipeline_params(config_path) logger.info("pretrained_emb {b}", b=config.net_params.pretrained_emb) device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') logger.info("Device is {device}", device=device) SRC, TRG, dataset = get_dataset(config.dataset_path, False) train_data, valid_data, test_data = split_data( dataset, **config.split_ration.__dict__) SRC.build_vocab(train_data, min_freq=3) TRG.build_vocab(train_data, min_freq=3) torch.save(SRC.vocab, config.src_vocab_name) torch.save(TRG.vocab, config.trg_vocab_name) logger.info("Vocab saved") print(f"Unique tokens in source (ru) vocabulary: {len(SRC.vocab)}") print(f"Unique tokens in target (en) vocabulary: {len(TRG.vocab)}") train_iterator, valid_iterator, test_iterator = BucketIterator.splits( (train_data, valid_data, test_data), batch_size=config.BATCH_SIZE, device=device, sort_key=_len_sort_key, ) INPUT_DIM = len(SRC.vocab) OUTPUT_DIM = len(TRG.vocab) config_encoder = BertConfig(vocab_size=INPUT_DIM) config_decoder = BertConfig(vocab_size=OUTPUT_DIM) config = EncoderDecoderConfig.from_encoder_decoder_configs( config_encoder, config_decoder) model = EncoderDecoderModel(config=config) config_encoder = model.config.encoder config_decoder = model.config.decoder config_decoder.is_decoder = True config_decoder.add_cross_attention = True config = EncoderDecoderConfig.from_encoder_decoder_configs( config_encoder, config_decoder) model = EncoderDecoderModel(config=config) args = TrainingArguments( output_dir="output", evaluation_strategy="steps", eval_steps=500, per_device_train_batch_size=128, per_device_eval_batch_size=128, num_train_epochs=10, save_steps=3000, seed=0, load_best_model_at_end=True, ) # args.place_model_on_device = device trainer = Trainer( model=model, args=args, train_dataset=train_iterator, eval_dataset=valid_iterator, callbacks=[EarlyStoppingCallback(early_stopping_patience=3)], ) trainer.train() model.save_pretrained("bert2bert")
def train_net(net, data, label, nepochs, batch_size, eval_single_output_fn = None): ''' Trains the given RCNN using criterion to calculate the loss and optimizer to adjust the weights. Prints out the training loss, training accuracy, and test accuracy every epoch. ''' print("batch_size:", batch_size) print("nepochs:", nepochs) for epoch in range(1,nepochs+1): # TrainX is a list of batches for training. Each batch if a vector of size # (batch_size, num_seq, embedding_length). # TrainY is a list of labels. # TestX and testY are of the same format for testing. trainX, trainY, testX, testY = load_data.split_data(data, label, batch_size) train_loss, train_acc = train_pattern(net, trainX, trainY, epoch) test_loss, test_acc = eval_model(net, testX, testY) print("epoch %s: train loss %2.2f train accuracy %2.2f\n test loss %2.2f test accuracy %2.2f" % (epoch, train_loss, train_acc, test_loss, test_acc))
def construct_model_and_data(args): """ Load model and data on which the attack is carried out. Assign target classes and images for targeted attack. """ data_model = args.dataset_name + args.model_name dataset = ImageData(args.dataset_name) x_test, y_test = dataset.x_val, dataset.y_val reference = - dataset.x_train_mean model = ImageModel(args.model_name, args.dataset_name, train = False, load = True) # Split the test dataset into two parts. # Use the first part for setting target image for targeted attack. x_train, y_train, x_test, y_test = split_data(x_test, y_test, model, num_classes = model.num_classes, split_rate = 0.5, sample_per_class = np.min([np.max([200, args.num_samples // 10 * 3]), 1000])) outputs = {'data_model': data_model, 'x_test': x_test, 'y_test': y_test, 'model': model, 'clip_max': 1.0, 'clip_min': 0.0 } if args.attack_type == 'targeted': # Assign target class and image for targeted atttack. label_train = np.argmax(y_train, axis = 1) label_test = np.argmax(y_test, axis = 1) x_train_by_class = [x_train[label_train == i] for i in range(model.num_classes)] target_img_by_class = np.array([x_train_by_class[i][0] for i in range(model.num_classes)]) np.random.seed(0) target_labels = [np.random.choice([j for j in range(model.num_classes) if j != label]) for label in label_test] target_img_ids = [np.random.choice(len(x_train_by_class[target_label])) for target_label in target_labels] target_images = [x_train_by_class[target_labels[j]][target_img_id] for j, target_img_id in enumerate(target_img_ids)] outputs['target_labels'] = target_labels outputs['target_images'] = target_images return outputs
def main(): start, end, seed = process_arguments.parse_arguments() dataframe = process_arguments.read_csv(start, end) column_names = [ "Country", "Year", "Region", "Happiness Score", "Economy (GDP per Capita)", "Health (Life Expectancy)", "Freedom", "Trust (Government Corruption)", "Generosity" ] labels, data = load_data.split_data(dataframe, column_names) # use the MinMaxScaler to scale all features between 0 and 1 scaler = MinMaxScaler() features_minmax = pd.DataFrame(data=data) features_minmax = scaler.fit_transform(features_minmax) data_train, data_test, labels_train, labels_test =\ load_data.split_train_test(features_minmax, labels, seed) # LINEAR REGRESSION MODEL print("LINEAR REGRESSION MODEL") linear_regression =\ LinearRegressionModel(data_test, labels_test, data_train, labels_train,\ start, end) linear_regression.train() linear_regression.predict() linear_regression.graph() print("R^2 = " + str(linear_regression.r_squared())) print("Weights = " + str(linear_regression.weights())) # KNEIGHBORS REGRESSOR MODEL print("\nKNEIGHBORS REGRESSOR MODEL") knregressor =\ KNeighborsRegressorModel(data_test, labels_test, data_train, labels_train,\ start, end) knregressor.train() knregressor.predict() knregressor.graph() print("R^2 = " + str(knregressor.r_squared()))
#!/usr/bin/env python3 """One-vs-one classifier.""" from sklearn.linear_model import SGDClassifier from sklearn.model_selection import cross_val_score from sklearn.multiclass import OneVsOneClassifier from sklearn.preprocessing import StandardScaler import numpy as np import load_data if __name__ == '__main__': train_data, train_label, test_data, test_label = load_data.split_data() ovo_clf = OneVsOneClassifier(SGDClassifier(random_state=42)) print( cross_val_score(ovo_clf, train_data, train_label, cv=3, scoring="accuracy")) scaler = StandardScaler() train_data_scaled = scaler.fit_transform(train_data.astype(np.float64)) print('After scaled: {}'.format( cross_val_score(ovo_clf, train_data_scaled, train_label, cv=3, scoring="accuracy")))
import cPickle import csv import timeit import matplotlib.pyplot as plt import numpy as np import theano import theano.tensor as T import ACNN import generate_function import load_data # load data train_x, train_y, test_x = load_data.train_test('./dataset/digit/train.csv', './dataset/digit/test.csv') train_x, train_y, valid_x, valid_y = load_data.split_data(train_x, train_y) train_x, train_y = load_data.shared_data(train_x, train_y) valid_x, valid_y = load_data.shared_data(valid_x, valid_y) X = T.matrix('input', dtype=theano.config.floatX) y = T.ivector('labels') index = T.lscalar('index') batch_size = 20 learning_rate = 0.01 train_batches = train_x.get_value(borrow=True).shape[0] // batch_size valid_batches = valid_x.get_value(borrow=True).shape[0] // batch_size rng = np.random.RandomState(1234) # create model layer0_input = X.reshape((batch_size, 1, 28, 28)) conv_net = ACNN.Al_cnn( input=layer0_input,
SEMANTIC_EMBED = 512 MAX_ITER = 100 batch_size = 128 image_size = 224 images, tags, labels = loading_data(DATA_DIR) dimTxt = tags.shape[1] dimLab = labels.shape[1] DATABASE_SIZE = 18015 TRAINING_SIZE = 10000 QUERY_SIZE = 2000 VERIFICATION_SIZE = 1000 X, Y, L = split_data(images, tags, labels, QUERY_SIZE, TRAINING_SIZE, DATABASE_SIZE) train_L = L['train'] train_x = X['train'] train_y = Y['train'] query_L = L['query'] query_x = X['query'] query_y = Y['query'] retrieval_L = L['retrieval'] retrieval_x = X['retrieval'] retrieval_y = Y['retrieval'] num_train = train_x.shape[0] numClass = train_L.shape[1] dimText = train_y.shape[1]
from sklearn.metrics import precision_recall_curve from sklearn.model_selection import cross_val_predict import matplotlib.pyplot as plt import load_data def plot_precision_recall_vs_threshold(precisions, recalls, thresholds): """Show precision_recall_curve.""" plt.plot(thresholds, precisions[:-1], "b--", label="Precision") plt.plot(thresholds, recalls[:-1], "g-", label="Recall") plt.xlabel("Threshold") plt.legend(loc="upper left") plt.ylim([0, 1]) if __name__ == '__main__': train_data, train_label, _, _ = load_data.split_data() train_label_5 = (train_label == 5) sgd_clf = SGDClassifier(random_state=42) scores = cross_val_predict(sgd_clf, train_data, train_label_5, cv=3, method="decision_function") precisions, recalls, thres = precision_recall_curve(train_label_5, scores) plot_precision_recall_vs_threshold(precisions, recalls, thres) plt.show()
## data data_view = ['CC', 'MLO'] data_type = ['Dense', 'Fatty'] type_v = 1 data, target = load_medical_data(data_type[type_v]) if scale_data: data = scale_data_normlize(data) feature_number = data[data_view[0]].shape[1] logger.info("data type {}".format(data_type[type_v])) kf = KFold(**kfold_params_dict) ## train experts = {} experts_split_indexes = split_data(data['CC'], is_multi_expert=False, kfold=kf) print "train experts" for i, view in enumerate(data_view): np.random.seed(check_seed) expert = expert_model(input_shape=(feature_number, ), index=view, params=current_params) expert.compile(optimizer=current_params['optimizer'], loss='binary_crossentropy', metrics=['accuracy']) #, fmeasure]) train_phase(expert, data[view], target, view, fit_params_dict, experts_split_indexes, False, params_dir) experts[view] = expert print "train combined models"
import pickle from load_data import split_data def import_model(): with open("model.pkl", 'rb') as f_un: model = pickle.load(f_un) return model if __name__ == '__main__': model = import_model() X = split_data(f_name='../data/data.json', for_predict=True): model.predict(X)
config = yaml.load(file, Loader=yaml.FullLoader)[config] date = datetime.now().strftime("%d%m%Y-%H%M") output = f"./output/{date}/" if args.path is None else args.path Path(output).mkdir(parents=True, exist_ok=True) #### Logger #### logging.basicConfig(filename=f"{output}log.txt", level=logging.INFO, format='%(asctime)s %(levelname)s: %(message)s', datefmt='%d.%m.%Y %H:%M:%S') #### Load Data #### x, y = load_data(**config["data"]) x = x.to_numpy().reshape(-1, config["causal_conv"]["c_in"], config["causal_conv"]["seq_length"]) y = y[::config["causal_conv"]["seq_length"]] # Create lagged outputs train, val = split_data(x, y.to_numpy()) train_loader = DataLoader(train, **config["loader"]) val_loader = DataLoader(val, **config["loader"]) #### Create Model ##### model = CausalConvNet(**config["causal_conv"]).cuda() classifier = MLP(num_class=np.max(y) + 1,**config["mlp"]).cuda() #### Tracking Varaibales ##### train_batch_losses = [] train_loss = [] train_metrics = [] train_pred = np.array([]) train_label = np.array([])
if 'data' not in os.listdir(data_model): os.mkdir('{}/data'.format(data_model)) if 'figs' not in os.listdir(data_model): os.mkdir('{}/figs'.format(data_model)) print('Loading dataset...') dataset = ImageData(args.dataset_name) model = ImageModel(args.model_name, args.dataset_name, train=False, load=True) if args.dataset_name == 'cifar10': X_train, Y_train, X_test, Y_test = split_data(dataset.x_val, dataset.y_val, model, num_classes=10, split_rate=0.8, sample_per_class=1000) print('Sanity checking...') data_sample = X_test print('data_sample.shape', data_sample.shape) print('X_train.shape', X_train.shape) pred_test = model.predict(dataset.x_val) def cross_entropy(predictions, targets, epsilon=1e-12): predictions = np.clip(predictions, epsilon, 1. - epsilon) N = predictions.shape[0] ce = -np.sum(targets * np.log(predictions + 1e-9)) / N return ce
import numpy as np import pandas as pd from models.oracle import Oracle from models.surrogate_teacher import Surrogate from models.omniscient_teacher import Omniscient from models.random_teacher import Random from models.without_teacher import Without_teacher from utils import predict, predict_by_W, rmse_W, write_np2csv, rmse_w, make_random_mask, predict_wj from load_data import read_W, read_csv, split_data from tqdm import tqdm from sklearn.metrics import roc_auc_score import logging import datetime # %% df = read_csv('output/wine-quality-pm1.csv', header=0) train_X, test_X, train_y, test_y = split_data(df, True) eta, lambd, alpha = 1, 2, 0.01 training_epochs, loops = 10, 10 J = 10 # 提示する教材合計数 textbook = 500 # 推定に使う教材数 test_textbook_list = [100] # 推定間に提示する教材数 between_textbook_list = [1] # 組 k = 1 lambds = [1, 2, 3, 4, 5] for lambd in lambds: oracle = Oracle(eta=eta, lambd=lambd)
import pandas as pd from models.oracle import Oracle from models.surrogate_teacher import Surrogate from models.omniscient_teacher import Omniscient from models.random_teacher import Random from models.without_teacher import Without_teacher from utils import predict, predict_by_W, rmse_W, write_np2csv, rmse_w, make_random_mask, predict_wj from load_data import read_W, read_csv, split_data from tqdm import tqdm from sklearn.metrics import roc_auc_score import logging import datetime # %% df = read_csv('output/weebil_vespula_pm1.csv', header=0) train_X, test_X, train_y, test_y = split_data(df, False) eta, lambd, alpha = 1, 2, 0.01 training_epochs, loops = 10, 10 J = 10 # 提示する教材合計数 textbook = 500 # 推定に使う教材数 test_textbook_list = [100] # 推定間に提示する教材数 between_textbook_list = [1] # 組 k = 1 lambds = [1, 2, 3, 4, 5] for lambd in lambds: oracle = Oracle(eta=eta, lambd=lambd)
def train_model(config_path: str): writer = SummaryWriter() config = read_training_pipeline_params(config_path) logger.info("pretrained_emb {b}", b=config.net_params.pretrained_emb) device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') logger.info("Device is {device}", device=device) SRC, TRG, dataset = get_dataset(config.dataset_path, config.net_params.transformer) train_data, valid_data, test_data = split_data( dataset, **config.split_ration.__dict__) if config.net_params.pretrained_emb: src_vectors = torchtext.vocab.FastText(language='ru') SRC.build_vocab(train_data, min_freq=3) if config.net_params.pretrained_emb: SRC.vocab.load_vectors(src_vectors) TRG.build_vocab(train_data, min_freq=3) torch.save(SRC.vocab, config.src_vocab_name) torch.save(TRG.vocab, config.trg_vocab_name) logger.info("Vocab saved") print(f"Unique tokens in source (ru) vocabulary: {len(SRC.vocab)}") print(f"Unique tokens in target (en) vocabulary: {len(TRG.vocab)}") train_iterator, valid_iterator, test_iterator = BucketIterator.splits( (train_data, valid_data, test_data), batch_size=config.BATCH_SIZE, device=device, sort_key=_len_sort_key, ) INPUT_DIM = len(SRC.vocab) OUTPUT_DIM = len(TRG.vocab) if config.net_params.attention: Encoder = network_gru_attention.Encoder Decoder = network_gru_attention.Decoder Seq2Seq = network_gru_attention.Seq2Seq Attention = network_gru_attention.Attention attn = Attention(config.net_params.HID_DIM, config.net_params.HID_DIM) enc = Encoder(INPUT_DIM, config.net_params.ENC_EMB_DIM, config.net_params.HID_DIM, config.net_params.HID_DIM, config.net_params.ENC_DROPOUT) dec = Decoder(OUTPUT_DIM, config.net_params.DEC_EMB_DIM, config.net_params.HID_DIM, config.net_params.HID_DIM, config.net_params.DEC_DROPOUT, attn) model = Seq2Seq(enc, dec, device) if config.net_params.transformer: logger.info("Transformer lets go") Encoder = network_transformer.Encoder Decoder = network_transformer.Decoder Seq2Seq = network_transformer.Seq2Seq SRC_PAD_IDX = SRC.vocab.stoi[SRC.pad_token] TRG_PAD_IDX = TRG.vocab.stoi[TRG.pad_token] HID_DIM = 512 ENC_LAYERS = 6 DEC_LAYERS = 6 ENC_HEADS = 8 DEC_HEADS = 8 ENC_PF_DIM = 2048 DEC_PF_DIM = 2048 ENC_DROPOUT = 0.1 DEC_DROPOUT = 0.1 enc = Encoder(INPUT_DIM, HID_DIM, ENC_LAYERS, ENC_HEADS, ENC_PF_DIM, ENC_DROPOUT, device) dec = Decoder(OUTPUT_DIM, HID_DIM, DEC_LAYERS, DEC_HEADS, DEC_PF_DIM, DEC_DROPOUT, device) model = Seq2Seq(enc, dec, SRC_PAD_IDX, TRG_PAD_IDX, device) if not config.net_params.attention and not config.net_params.transformer: Encoder = my_network.Encoder Decoder = my_network.Decoder Seq2Seq = my_network.Seq2Seq enc = Encoder(INPUT_DIM, config.net_params.ENC_EMB_DIM, config.net_params.HID_DIM, config.net_params.N_LAYERS, config.net_params.ENC_DROPOUT) dec = Decoder(OUTPUT_DIM, config.net_params.DEC_EMB_DIM, config.net_params.HID_DIM, config.net_params.N_LAYERS, config.net_params.DEC_DROPOUT) model = Seq2Seq(enc, dec, device) model.apply(init_weights) if config.net_params.pretrained_emb: model.encoder.tok_embedding = nn.Embedding.from_pretrained( torch.FloatTensor(SRC.vocab.vectors)) model.to(device) PAD_IDX = TRG.vocab.stoi[TRG.pad_token] optimizer = optim.Adam(model.parameters(), config.lr) criterion = nn.CrossEntropyLoss(ignore_index=PAD_IDX) lr_scheduler = optim.lr_scheduler.ReduceLROnPlateau( optimizer, **config.lr_scheduler.__dict__) train_history = [] valid_history = [] best_valid_loss = float('inf') print("Let's go") # for p in model.encoder.parameters(): # p.requires_grad = True # for p in model.decoder.parameters(): # p.requires_grad = True for epoch in range(config.N_EPOCHS): start_time = time.time() train_loss = train(model, train_iterator, optimizer, criterion, config.CLIP, train_history, valid_history) valid_loss = evaluate(model, valid_iterator, criterion) lr_scheduler.step(valid_loss) end_time = time.time() epoch_mins, epoch_secs = epoch_time(start_time, end_time) if valid_loss < best_valid_loss: best_valid_loss = valid_loss torch.save(model.state_dict(), config.model_out_name) train_history.append(train_loss) valid_history.append(valid_loss) writer.add_scalar('train loss', train_history[-1], epoch) writer.add_scalar('valid loss', valid_history[-1], epoch) writer.add_scalar('learning rate', lr_scheduler._last_lr[0], epoch) print(f'Epoch: {epoch + 1:02} | Time: {epoch_mins}m {epoch_secs}s') print( f'\tTrain Loss: {train_loss:.3f} | Train PPL: {math.exp(train_loss):7.3f}' ) print( f'\t Val. Loss: {valid_loss:.3f} | Val. PPL: {math.exp(valid_loss):7.3f}' ) for idx, batch in enumerate(valid_iterator): if idx > 3: break src = batch.src[:, idx:idx + 1] trg = batch.trg[:, idx:idx + 1] generate_translation(src, trg, model, TRG.vocab, SRC.vocab, config.net_params.transformer) get_bleu(model, test_iterator, TRG, config.net_params.transformer)
#!/usr/bin/env python3 """Multi output classification.""" from sklearn.neighbors import KNeighborsClassifier import matplotlib import matplotlib.pyplot as plt import numpy as np import load_data if __name__ == '__main__': train_data, _, test_data, _ = load_data.split_data() train_mod = train_data + np.random.randint(0, 100, (len(train_data), 784)) test_mod = test_data + np.random.randint(0, 100, (len(test_data), 784)) knn_clf = KNeighborsClassifier() knn_clf.fit(train_mod, train_data) random_index = 100 clean_digit = knn_clf.predict([test_mod[random_index]]) plt.subplot(121) src_image = test_mod[random_index].reshape(28, 28) plt.imshow(src_image, cmap=matplotlib.cm.binary, interpolation="nearest") plt.subplot(122) clean_image = clean_digit.reshape(28, 28) plt.imshow(clean_image, cmap=matplotlib.cm.binary, interpolation="nearest") plt.show()