def run_resnet2(t, v): # All layers frozen, except fc tfms = transforms.Compose([ transforms.Resize((image_size, image_size)), # transforms.CenterCrop(params['image_size']), transforms.ToTensor(), # Converts a PIL Image or numpy.ndarray (H x W x C) in the range [0, 255] to a torch.FloatTensor of shape (C x H x W) in the range [0.0, 1.0] transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) ]) train_items, train_labels = t['id_aug'].values, t['label'].values valid_items, valid_labels = v['id_aug'].values, v['label'].values train_ds = ImageDataset(train_items, train_labels, tfms) valid_ds = ImageDataset(valid_items, valid_labels, tfms) net = Resnet(dropout=0.) for layer in [ net.resnet.layer1, net.resnet.layer2, net.resnet.layer3, net.resnet.layer4 ]: for param in layer.parameters(): param.requires_grad = False print_model_details(net, params) optimizer = Adam(net.parameters(), lr=lr) criterion = nn.CrossEntropyLoss() trainer = Trainer(train_ds=train_ds, valid_ds=valid_ds, model=net, criterion=criterion, optimizer=optimizer, params=params, callbacks=cbc, metrics=mc) trainer.train()
def test_train_method(self): file_name = 'test/test_data/attention_test.txt' fine_tune_model_name = '../models/glove_model_40.pth' self.test_data_loader_attention = DataLoaderAttention( file_name=file_name) self.test_data_loader_attention.load_data() source2index, index2source, target2index, index2target, train_data = \ self.test_data_loader_attention.load_data() HIDDEN_SIZE = 512 NUM_LAYERS = 2 KERNEL_SIZE = 2 EMBEDDING_SIZE = 50 SOURCE_VOCAB_SIZE = len(source2index) TARGET_VOCAB_SIZE = len(target2index) qrnn = QRNNModel(QRNNLayer, NUM_LAYERS, KERNEL_SIZE, HIDDEN_SIZE, EMBEDDING_SIZE, SOURCE_VOCAB_SIZE, TARGET_VOCAB_SIZE) self.trainer = Trainer(epoch=100, fine_tune_model=fine_tune_model_name) self.trainer.train_qrnn( train_data=train_data, source2index=source2index, target2index=target2index, index2source=index2source, index2target=index2target, qrnn_model=qrnn, )
def test_train_method(self): file_name = 'test/test_data/attention_test.txt' fine_tune_model_name = '../models/glove_model_40.pth' self.test_data_loader_attention = DataLoaderAttention( file_name=file_name) self.test_data_loader_attention.load_data() source2index, index2source, target2index, index2target, train_data = \ self.test_data_loader_attention.load_data() EMBEDDING_SIZE = 50 HIDDEN_SIZE = 32 encoder = Encoder(len(source2index), EMBEDDING_SIZE, HIDDEN_SIZE, 3, True) decoder = Decoder(len(target2index), EMBEDDING_SIZE, HIDDEN_SIZE * 2) self.trainer = Trainer(fine_tune_model=fine_tune_model_name) self.trainer.train_attention( train_data=train_data, source2index=source2index, target2index=target2index, index2source=index2source, index2target=index2target, encoder_model=encoder, decoder_model=decoder, )
def test_train_method(self): self.test_data_loader = DataLoader() self.test_japanese_wiki_data = 'test/test_data/jawiki_test.txt' test_word2index, test_index2word, test_window_data, \ test_X_ik, test_weightinhg_dict = self.test_data_loader.load_data(file_name=self.test_japanese_wiki_data) # noqa self.test_prepare_train_data = PrepareTrainData() test_train_data = \ self.test_prepare_train_data.prepare_train_data_method( window_data=test_window_data, word2index=test_word2index, weighting_dic=test_weightinhg_dict, X_ik=test_X_ik) self.model = Glove(vocab_size=len(test_word2index)) self.trainer = Trainer(model=self.model) self.trainer.train_method(train_data=test_train_data) word_similarity = self.trainer.word_similarity( target=self.test_data_loader.vocab[0], vocab=self.test_data_loader.vocab, word2index=test_word2index, top_rank=2) word_similarity_check = ['<', '>', 's'] word_similarity_bool = False for word in word_similarity: if word[0] in word_similarity_check: word_similarity_bool = True assert word_similarity_bool is True
def main(): parser = argparse.ArgumentParser(description="Training attention model") parser.add_argument( "-t", "--train_data", metavar="train_data", type=str, default='../data/processed/source_replay_twitter_data.txt', dest="train_data", help="set the training data ") parser.add_argument("-e", "--embedding_size", metavar="embedding_size", type=int, default=50, dest="embedding_size", help="set the embedding size ") parser.add_argument("-H", "--hidden_size", metavar="hidden_size", type=int, default=512, dest="hidden_size", help="set the hidden size ") parser.add_argument("-f", "--fine_tune_model_name", metavar="fine_tune_model_name", type=str, default='../models/glove_wiki/glove_model_40.pth', dest="fine_tune_model_name", help="set the fine tune model name ") args = parser.parse_args() data_loader_attention = DataLoaderAttention(file_name=args.train_data) data_loader_attention.load_data() source2index, index2source, target2index, index2target, train_data = \ data_loader_attention.load_data() EMBEDDING_SIZE = args.embedding_size HIDDEN_SIZE = args.hidden_size encoder = Encoder(len(source2index), EMBEDDING_SIZE, HIDDEN_SIZE, 3, True) decoder = Decoder(len(target2index), EMBEDDING_SIZE, HIDDEN_SIZE * 2) trainer = Trainer(epoch=600, batch_size=64, fine_tune_model=args.fine_tune_model_name) trainer.train_attention(train_data=train_data, source2index=source2index, target2index=target2index, index2source=index2source, index2target=index2target, encoder_model=encoder, decoder_model=decoder)
def train(): opt = TrainOptions().parse() if opt.distributed: init_dist() opt.batchSize = opt.batchSize // len(opt.gpu_ids) ### setup dataset data_loader = CreateDataLoader(opt) dataset = data_loader.load_data() pose = 'pose' in opt.dataset_mode ### setup trainer trainer = Trainer(opt, data_loader) ### setup models model, flowNet = create_model(opt, trainer.start_epoch) flow_gt = conf_gt = [None] * 2 for epoch in range(trainer.start_epoch, opt.niter + opt.niter_decay + 1): trainer.start_of_epoch(epoch, model, data_loader) n_frames_total, n_frames_load = data_loader.dataset.n_frames_total, opt.n_frames_per_gpu for idx, data in enumerate(dataset, start=trainer.epoch_iter): trainer.start_of_iter() if not opt.no_flow_gt: data_list = [ data['tgt_label'], data['ref_label'] ] if pose else [data['tgt_image'], data['ref_image']] flow_gt, conf_gt = flowNet(data_list, epoch) data_list = [ data['tgt_label'], data['tgt_image'], flow_gt, conf_gt ] data_ref_list = [data['ref_label'], data['ref_image']] data_prev = [None, None] ############## Forward Pass ###################### for t in range(0, n_frames_total, n_frames_load): data_list_t = get_data_t(data_list, n_frames_load, t) + data_ref_list + data_prev g_losses, generated, data_prev = model( data_list_t, save_images=trainer.save, mode='generator') g_losses = loss_backward(opt, g_losses, model.module.optimizer_G) d_losses = model(data_list_t, mode='discriminator') d_losses = loss_backward(opt, d_losses, model.module.optimizer_D) loss_dict = dict( zip(model.module.lossCollector.loss_names, g_losses + d_losses)) if trainer.end_of_iter(loss_dict, generated + data_list + data_ref_list, model): break trainer.end_of_epoch(model)
def test_train(self): env = Env(balance=250000, FX_DATA_FILE='../data/raw/FX_Demo/sample_USD_JPY_S5.pickle') agent = Agent(input_data_shape=(10, )) mount_agent = Agent(actions=10, input_data_shape=(10, )) print(len(env.fx_time_data_buy)) trainer = Trainer(env, agent, mount_agent, Adam(lr=1e-6), data_end_index=len(env.fx_time_data_buy) - 2) trainer.train()
def main(): parser = argparse.ArgumentParser() parser.add_argument('--config', type=str, default="config/base.yml") args = parser.parse_args() print("="*30 + "CONFIG INFOR" + "="*30) config = Cfg.load_config_from_file(args.config) print(config) trainer = Trainer(config) trainer.train()
def main(): #load logging if constants.ENABLE_LOGGING == True: load_logging() else: logging.getLogger().disabled = True #load the config CONFIG = load_config(constants.CONFIG_PATH) #check if cuda is available & is enabled in config constants.USE_CUDA = CONFIG['use_cuda'] & torch.cuda.is_available() if constants.USE_CUDA == True: constants.DEVICE = 'cuda:0' else: constants.DEVICE = 'cpu' print(constants.DEVICE) print(torch.cuda.is_available()) #get the data generators vocab, train_generator, validation_generator = load_data( constants.DATASET_PATH, CONFIG) #use incremented vocabulary size because we have an extra character which isn't #in the dictionary : 0 - padding character model = MyModel(vocab_size=vocab.size() + 1, embedding_size=CONFIG['embedding_size'], rnn_size=CONFIG['rnn_size'], output_size=CONFIG['output_size']) model = model.to(constants.DEVICE) trainer = Trainer(model=model, vocab=vocab, train_generator=train_generator, val_generator=validation_generator, epochs=CONFIG['epochs'], batch_size=CONFIG['batch_size'], max_grad_norm=CONFIG['max_grad_norm'], lr=CONFIG['learning_rate'], loss=CONFIG['loss'], optim=CONFIG['optimizer'], train_verbose=CONFIG['train_verbose'], val_verbose=CONFIG['validation_verbose']) trainer.train() if CONFIG['save_model'] == True: save_model(model)
def select(id): sql = "SELECT * FROM Trainers WHERE id = %s" values = [id] sql_result = run_sql(sql, values)[0] trainer = Trainer(sql_result['name'], sql_result['email'], sql_result['phone'], sql_result['specialism'], sql_result['id']) return trainer
def update_trainer(id): name = request.form["name"] email = request.form["email"] phone = request.form["phone"] specialism = request.form["specialism"] updated_trainer = Trainer(name, email, phone, specialism, id) trainer_repository.update(updated_trainer) return redirect("/trainers")
def select_all(): trainers = [] sql = 'SELECT * FROM trainers' results = run_sql(sql) for row in results: trainer = Trainer(row['name'], row['active'], row['id']) trainers.append(trainer) return trainers
def create_trainer(): name = request.form["name"] email = request.form["email"] phone = request.form["phone"] specialism = request.form["specialism"] new_trainer = Trainer(name, email, phone, specialism) trainer_repository.save(new_trainer) return redirect("/trainers")
def trainer(): return Trainer( name='Sebastian', lastName='Herrera', age=21, phone=632432654, email='*****@*****.**' )
def select(id): trainer = None sql = "SELECT * FROM trainers WHERE id = %s" values = [id] result = run_sql(sql, values)[0] if result is not None: trainer = Trainer(result['name'], result['number'], result['id']) return trainer
def main(): parser = argparse.ArgumentParser(description="Training glove model") parser.add_argument( "-c", "--train_data", metavar="train_data", # type=str, default='../data/raw/jawiki_only_word_random_choose.txt', type=str, default='../data/raw/source_replay_twitter_data_sort.txt', dest="train_data", help="set the training data ") parser.add_argument("-e", "--embedding_size", metavar="embedding_size", type=int, default=300, dest="embedding_size", help="set the embedding size") args = parser.parse_args() data_loader = DataLoader() japanese_wiki_data = args.train_data word2index, index2word, window_data, X_ik, weightinhg_dict = \ data_loader.load_data(file_name=japanese_wiki_data) # noqa print(word2index) prepare_train_data = PrepareTrainData() train_data = \ prepare_train_data.prepare_train_data_method( window_data=window_data, word2index=word2index, weighting_dic=weightinhg_dict, X_ik=X_ik) model = Glove(vocab_size=len(word2index), projection_dim=args.embedding_size) trainer = Trainer(model=model) trainer.train_method(train_data=train_data) word_similarity = trainer.word_similarity(target=data_loader.vocab[0], vocab=data_loader.vocab, word2index=word2index, top_rank=2) print(word_similarity)
def select(id): trainer = None sql = "SELECT * FROM trainers WHERE id = %s" values = [id] result = run_sql(sql, values)[0] if result is not None: location = Trainer(result['first_name'], result['last_name'], result['id'] ) return location
def select_all(): trainers = [] sql = "SELECT * FROM trainers" results = run_sql(sql) for sql_result in results: trainer = Trainer(sql_result['name'], sql_result['email'], sql_result['phone'], sql_result['specialism'], sql_result['id']) trainers.append(trainer) return trainers
def select_all(): trainers = [] sql = "SELECT * FROM trainers" results = run_sql(sql) for row in results: trainer = Trainer(row['first_name'], row['last_name'], row['id']) trainers.append(trainer) return trainers
def main(config): # Load data loaders and model device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') loader_train, loader_test, loader_valid = get_dataloaders( **config["dataloader"]) if config["use_pretrained_model"]: model = build_pretrained_model(config["model_name"]) else: model = MyCNN(im_size=config["dataloader"]["im_size"], config_block=config["model"]["block"], n_out_channels=config["model"]["n_out_channels"], n_hidden_neurons=config["model"]["n_hidden_neurons"]) # Train model trainer = Trainer( model=model, loader_train=loader_train, loader_valid=loader_valid, params_optim=config["params_optim"], params_lr_scheduler=None, # config["params_lr_scheduler"], device=device, **config["trainer"]) trainer.train(n_iter=config["train"]["n_iter"]) # Test model on new observations (to see model's performance in a real life scenario) conf_mat_test = get_confusion_matrix( model=trainer.model, threshold=config["trainer"]["threshold"], data_loader=loader_test, device=device) f1_test = round(get_f1_score(conf_mat_test), 3) acc_test = round(get_accuracy(conf_mat_test), 3) print("On the test set, F1 score = {} and accuracy = {}".format( f1_test, acc_test)) # Explain some predictions with SHAP values e = Explainer(model, loader_train, device, savepath="./saved/figures") images_test, label_test = sample_data(loader_test, 5) e.fit() e.explain(images_test, label_test)
def main(): parser = argparse.ArgumentParser( description='Execute train reinforcement learning.') parser.add_argument( '--dataset_name', type=str, default="../data/raw/FX_Demo/sample10000_USD_JPY_S5.pickle", help='an integer for the accumulator') args = parser.parse_args() print(args.dataset_name) env = Env(balance=250000, FX_DATA_FILE=args.dataset_name) agent = Agent(input_data_shape=(10, )) mount_agent = Agent(actions=10, input_data_shape=(10, )) trainer = Trainer(env, agent, mount_agent, Adam(lr=1e-6), data_end_index=len(env.fx_time_data_buy) - 2) trainer.train()
def main(): parser = argparse.ArgumentParser() parser.add_argument("--config", default="./configs/" + CONFIG_FILE) option = parser.parse_args() config = json.load(open(option.config)) model_name = config["model"] features = config["features"] target_name = config["target_name"] params = config["model_params"] #lineNotify("leanrning start") X, y, test = load_data(features, target_name) #logger get_logger(VERSION).info(test.shape) get_logger(VERSION).info(option.config) get_logger(VERSION).info(params) get_logger(VERSION).info("====== CV score ======") trainer = Trainer(model_name, params, features) val_score, val_log = trainer.cross_validation(X, y, test, random_state=RS, n_split=K, importance_f=imp, task=TASK) get_logger(VERSION).info(np.mean(val_score["lgb"])) get_logger(VERSION).info(val_log) # prediction preds = trainer.predict() #submit trainer.create_submit(preds, val_score)
def test_model(): train_dataset = torch.load( os.path.join(paths.experiment_data_dir, 'train.data')) dev_dataset = torch.load( os.path.join(paths.experiment_data_dir, 'dev.data')) test_dataset = torch.load( os.path.join(paths.experiment_data_dir, 'test.data')) vocab = torch.load(paths.vocab_path) model = RelationClassifier(encoder_type=args.encoder, num_classes=len(train_dataset.label_map), vocab_size=vocab.size(), embed_dim=vocab.embed_dim, model_config=ModelConfig, drop_rate=args.drop, use_cuda=args.cuda) model.load_state_dict(torch.load(paths.best_model_path)) trainer = Trainer(model, use_cuda=args.cuda) # train_loss, train_acc, train_f1 = trainer.eval(train_dataset, vocab, 'trainset') # print('Train: loss {}, acc {}, f1 {}'.format(train_loss, train_acc, train_f1)) dev_acc = trainer.eval(dev_dataset, vocab, 'devset') print('Dev: acc {}'.format(dev_acc)) test_acc = trainer.eval(test_dataset, vocab, 'testset') print('Test: acc {}'.format(test_acc))
def training(ngrams_path, resume=False, checkpoint_path=""): list_ngrams = load_dataset(ngrams_path) trainer = Trainer(alphabets, list_ngram=list_ngrams) # Resume training if resume: trainer.load_checkpoint(checkpoint_path) # Training trainer.train()
def main(_): # creating the directories for model weights, # logs and tensorboard logs prepare_dirs(config) rng = np.random.RandomState(config.random_seed) tf.set_random_seed(config.random_seed) read_semantics = config.read_semantics if read_semantics and not config.semantic_regularizer: model = DsNet(config) print("using DS-net architecture!") elif config.semantic_regularizer: model = DepthModelRegularized(config) print("using Depth-net regularized architecture!") else: model = DepthModel(config) print("using depth-net architecture!") # creating the dataset and the data loader objects dataset = Dataset(config=config, name="kitti", read_semantics=read_semantics) data_loader = DataLoader(dataset=dataset, shuffle_data=True) # creating our trainer trainer = Trainer(model=model, data_loader=data_loader, config=config, rng=rng) # training session if config.is_train: if config.resume_training: trainer.resume_training() else: trainer.train() # test session else: trainer.test(show_figure=False, save_figures=True, is_real_kitti=True)
def train_model(): train_dataset = torch.load( os.path.join(paths.experiment_data_dir, 'train.data')) dev_dataset = torch.load( os.path.join(paths.experiment_data_dir, 'dev.data')) test_dataset = torch.load( os.path.join(paths.experiment_data_dir, 'test.data')) vocab = torch.load(paths.vocab_path) model = RelationClassifier(encoder_type=args.encoder, num_classes=len(train_dataset.label_map), vocab_size=vocab.size(), embed_dim=vocab.embed_dim, model_config=ModelConfig, drop_rate=args.drop, use_cuda=args.cuda, attention=args.attention) # init with pre-trained embeddings model.argument_encoder.emb.weight.data.copy_(vocab.embeddings) # model.argument_encoder.emb.weight.requires_grad = False criterion = nn.CrossEntropyLoss() params_to_train = filter(lambda p: p.requires_grad, model.parameters()) if args.cuda: model.cuda(), criterion.cuda() if args.optim == 'sgd': optimizer = optim.SGD(params_to_train, lr=args.lr, weight_decay=args.wd) elif args.optim == 'adam': optimizer = optim.Adam(params_to_train, lr=args.lr, weight_decay=args.wd) elif args.optim == 'adagrad': optimizer = optim.Adagrad(params_to_train, lr=args.lr, weight_decay=args.wd) elif args.optim == 'rprop': optimizer = optim.Rprop(params_to_train, lr=args.lr) trainer = Trainer(model, criterion, optimizer, args.cuda) best_dev_acc = 0 for epoch in range(args.epochs): print() trainer.train(train_dataset, vocab, batch_size=args.batch_size) # train_loss, train_acc, train_f1 = trainer.eval(train_dataset, vocab, 'trainset') # print('Train: loss {}, acc {}, f1 {}'.format(train_loss, train_acc, train_f1)) dev_acc = trainer.eval(dev_dataset, vocab, 'devset') print('Dev: acc {}'.format(dev_acc)) test_acc = trainer.eval(test_dataset, vocab, 'testset') print('Test: acc {}'.format(test_acc)) if dev_acc > best_dev_acc: best_dev_acc = dev_acc print('Model saved to {}'.format(paths.best_model_path)) torch.save(model.state_dict(), paths.best_model_path)
from models.nurse import Nurse from models.pokemon import Pokemon from models.trainer import Trainer import repositories.nurse_repository as nurse_repository import repositories.pokemon_repository as pokemon_repository import repositories.trainer_repository as trainer_repository # pokemon_repository.delete_all() # nurse_repository.delete_all() # trainer_repository.delete_all() nurse_1 = Nurse("Joy", "Fire") nurse_repository.save(nurse_1) trainer = Trainer("Simon", 12345) trainer_repository.save(trainer) pokemon = Pokemon('Seed Backman', 'Bulbasaur', 'Grass', '25/09/1990', trainer , 'burned') pokemon.assign_nurse(nurse_1) pokemon_repository.save(pokemon) trainer.name = 'Simondovich' trainer_repository.update(trainer) nurse_1.name = "Betsy"
sequenceSize=settings.sequenceSize, stepSize=settings.stepSize, targetType="float", sensors=settings.sensors) print("Dataset read in time", time.time() - dsTime) print("Trainset size:", len(trainingSet), "Testset size:", len(testSet)) print("Trainging on", [x[2] for x in trainingSet]) print("Testing on", [x[2] for x in testSet]) module = SingleGRU() if settings.loadModel: module.load_state_dict(torch.load(settings.modelPath)) trainer = Trainer(module=module, optimizer=module.optimizer, learningRate=module.lr, lossFunction=module.lossFunction) if not settings.loadModel: trainer.train(trainingSet, testSet, module.numEpochs) torch.save(module.state_dict(), module.modelPath) print("remember to rename model file if wanted") trainer.printBenchmarks(trainingSet + testSet) trainer.storeBenchmarks("benchmarks" + module.modelPath[:-3] + ".txt") print("possible scenarios to store:") scenarios = [x[2] for x in (trainingSet + testSet)] print(scenarios) scen = input("which scenario should be stored?") while ("Scenario-" + scen) not in scenarios: scen = input("which scenario should be stored?")
from models.trainer import Trainer config = {"agent": "A3C", "episodes": 100} if __name__ == "__main__": trainer = Trainer(config) trainer.train()
seq2seq = Seq2seq(config, len(src.vocab), len(tgt.vocab), tgt.sos_id, tgt.eos_id, pretrained_pos_weight) else : seq2seq = Seq2seq(config, len(src.vocab), len(tgt.vocab), tgt.sos_id, tgt.eos_id) if torch.cuda.is_available(): seq2seq.cuda() for param in seq2seq.parameters(): param.data.uniform_(-0.08, 0.08) # train t = Trainer(loss=loss, batch_size=128, learning_rate=0.001, checkpoint_every=50, print_every=100, hidden_size=config["hidden_size"], path=save_path, file_name=config["rnn_cell"] + "_" + str(i)) seq2seq, ave_loss, character_accuracy_list, sentence_accuracy_list, f1_score_list = t.train(seq2seq, train, num_epochs=epochs, dev_data=dev, optimizer=optimizer, teacher_forcing_ratio=0.5) character_accuracy.append(character_accuracy_list) sentence_accuracy.append(sentence_accuracy_list) f1_score.append(f1_score_list) best_f1_score.append(max(f1_score_list)) # svae plot