def train(params): assert params["mode"].lower() == "train", "change training mode to 'train'" print("Creating the vocab from :", params["vocab_path"]) vocab = Vocab(params["vocab_path"], params["vocab_size"]) print("Creating the embedding_matrix from:", params["vector_path"]) embeddings_matrix = get_embedding(params["vocab_size"], params["embed_size"], vocab, params['vector_path']) tf.compat.v1.logging.info("Building the model ...") model = PGN(params, embeddings_matrix) print("Creating the batcher ...") b = batcher(params["data_dir"], vocab, params) print("Creating the checkpoint manager") checkpoint_dir = "{}".format(params["checkpoint_dir"]) ckpt = tf.train.Checkpoint(step=tf.Variable(0), PGN=model) ckpt_manager = tf.train.CheckpointManager(ckpt, checkpoint_dir, max_to_keep=11) ckpt.restore(ckpt_manager.latest_checkpoint) if ckpt_manager.latest_checkpoint: print("Restored from {}".format(ckpt_manager.latest_checkpoint)) else: print("Initializing from scratch.") tf.compat.v1.logging.info("Starting the training ...") train_model(model, b, params, ckpt, ckpt_manager, "output.txt")
def main(): # Basic usage: python train.py data_directory parser = argparse.ArgumentParser(description='Optional add-ons.') parser.add_argument('data_directory') parser.add_argument('--save_dir', action='store') parser.add_argument('--arch', action='store') parser.add_argument('--learning_rate', action='store', type=float) parser.add_argument('--hidden_units', action='store', type=int) parser.add_argument('--epochs', action='store', type=int) parser.add_argument('--gpu', action='store_true') args = parser.parse_args() # get category class mapping to names for flowers with open('cat_to_name.json', 'r') as f: cat_to_name = json.load(f) # capture train and validation data and transform train_data, valid_data = th.load_data(args.data_directory) trainloader, validloader, class_to_idx = th.transform_data( train_data, valid_data) # Use GPU if it's available device = torch.device('cuda') if ( args.gpu | torch.cuda.is_available()) else torch.device('cpu') print('Building model...') # set default to vgg11 args.arch = args.arch if args.arch else 'vgg11' model = models.__dict__[args.arch](pretrained=True) model.arch = args.arch # update model classifier to customize for our problem (102 category probability calcs) hidden_units = args.hidden_units if args.hidden_units else 4098 model.classifier = th.update_classifier(model, hidden_units) print('Training model...') epochs = args.epochs if args.epochs else 5 learning_rate = args.learning_rate if args.learning_rate else 0.001 th.train_model(model, trainloader, validloader, epochs, learning_rate, device) # add class_to_idx attribute to model model.class_to_idx = class_to_idx print('Saving model...') save_dir = args.save_dir if args.save_dir else 'saved_models' th.save_model(model, save_dir) print('Trained model saved!')
def train(params): # 1. 配置计算资源 config_gpus() # 2. vocab vocab = Vocab(params['vocab_path'], params['vocab_size']) params['vocab_size'] = vocab.count # 3. 构造模型 model = Seq2Seq(params, vocab) # 确保传入到模型的参数params的所有制不会再被修改,不然会报错。 # # 4. 模型存储 checkpoint = tf.train.Checkpoint(Seq2Seq=model) checkpoint_manager = tf.train.CheckpointManager( checkpoint, directory=path.join(params['checkpoint_dir'], 'seq2seq_model'), max_to_keep=5) if params['restore']: checkpoint.restore(checkpoint_manager.latest_checkpoint) # 5. 模型训练 train_model(model, vocab, params, checkpoint_manager)
exit(0) else: device = 'cpu' print("Further training will be done on cpu, switch to GPU\n") print("Selected Device: ", device, "\n") # load the datasets data, loader = load_data(data_dir) # make model model = make_model(arch, hidden_units, drop) model.to(device) # set optimizer state according to arch if model.name == 'vgg16' or model.name == 'densenet121': optimizer = optim.Adam(model.classifier.parameters(), lr=lr) elif model.name == 'resnet50': optimizer = optim.Adam(model.fc.parameters(), lr=lr) # train model, get new state of optimizer in order to save it in checkpoint trained_model, optimizer = train_model(model, optimizer, epochs, device, data, loader) # check accuracy compute_accuracy(trained_model, loader, device) # save model save_checkpoint(model , arch, optimizer, lr, epochs, hidden_units, drop, save_dir)
help='criterio de solución') parser.add_argument('--random_state', type=int, default=42, help='Entero aleatorio') parser.add_argument('--class_weight', type=int, default='balanced', help='data balanceada') args = parser.parse_args() run = Run.get_context() ws = run.experiment.workspace datastore = ws.get_default_datastore() input_ds = Dataset.get_by_name(ws, 'moa_ds') data = input_ds.to_pandas_dataframe() dataframe = validate_data(data) X_train, X_test, y_train, y_test = split_data(dataframe) model = train_model(X_train, y_train, save=True, criterion=args.criterion, random_state=args.random_state, class_weight=args.class_weight) y_pred = model.predict(X_test) y_prob = model.predict_proba(X_test) print(f"balanced_accuracy: {balanced_accuracy_score(y_test, y_pred)}") print(f"log_loss: {log_loss(y_test, y_prob)}") run.log('balanced_accuracy', balanced_accuracy_score(y_test, y_pred))
from sklearn.metrics import accuracy_score from train_helper import validate_data, split_data, train_model from sklearn.metrics import log_loss, balanced_accuracy_score from azureml.core import Run if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument('--data_path', type=str, default="./data/", help='Path to the training data') parser.add_argument('--file_name', type=str, default="dataset.csv", help='Filename') args = parser.parse_args() print("===== DATA =====") print("DATA PATH: " + args.data_path) print("FILE NAME: " + args.file_name) print("LIST FILES IN DATA PATH...") print(os.listdir(args.data_path)) print("================") data = pd.read_csv(args.data_path + args.file_name) datos = validate_data(data) X_train, y_train, X_test, y_test = split_data(datos) model = train_model(X_train, y_train, save=True) y_pred = model.predict(X_test) y_prob = model.predict_proba(X_test) print(f"balanced_accuracy: {balanced_accuracy_score(y_test, y_pred)}") print(f"log_loss: {log_loss(y_test, y_prob)}")
#!/usr/bin/env python import train_helper n_epochs = 20 batch_size = 32 dataloaders, class_names = train_helper.load_data('../../data', batch_size=batch_size) model, criterion, optimizer, scheduler = train_helper.get_model( dataloaders, n_epochs) model = train_helper.train_model(model, criterion, optimizer, scheduler, dataloaders, n_epochs=n_epochs) train_helper.save_model(model, 'model.pt') train_helper.save_mobile_model(model, 'model_mobile.pt')
torch.utils.data.DataLoader(image_datasets["train"], batch_size=64, shuffle=True), "test": torch.utils.data.DataLoader(image_datasets["test"], batch_size=64), "valid": torch.utils.data.DataLoader(image_datasets["valid"], batch_size=64) } if in_args.gpu: device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") if (device == "cpu"): print("not found gpu,.... running on cpu") else: device = "cpu" if (in_args.arch == "densenet"): model = thelp.densenet121_FlowerModel(in_args.nhu) else: model = thelp.vgg11_FlowerModel(in_args.nhu) criterion = nn.NLLLoss() optimizer = optim.Adam(model.classifier.parameters(), lr=in_args.lr) print(in_args) thelp.train_model(model, dataloaders, criterion, optimizer, image_datasets, device=device, epochs=in_args.epochs, nhu=in_args.nhu)