def train(params):
assert params["mode"].lower() == "train", "change training mode to 'train'"
print("Creating the vocab from :", params["vocab_path"])
vocab = Vocab(params["vocab_path"], params["vocab_size"])
print("Creating the embedding_matrix from:", params["vector_path"])
embeddings_matrix = get_embedding(params["vocab_size"],
params["embed_size"], vocab,
params['vector_path'])
tf.compat.v1.logging.info("Building the model ...")
model = PGN(params, embeddings_matrix)
print("Creating the batcher ...")
b = batcher(params["data_dir"], vocab, params)
print("Creating the checkpoint manager")
checkpoint_dir = "{}".format(params["checkpoint_dir"])
ckpt = tf.train.Checkpoint(step=tf.Variable(0), PGN=model)
ckpt_manager = tf.train.CheckpointManager(ckpt,
checkpoint_dir,
max_to_keep=11)
ckpt.restore(ckpt_manager.latest_checkpoint)
if ckpt_manager.latest_checkpoint:
print("Restored from {}".format(ckpt_manager.latest_checkpoint))
else:
print("Initializing from scratch.")
tf.compat.v1.logging.info("Starting the training ...")
train_model(model, b, params, ckpt, ckpt_manager, "output.txt")
def main():
# Basic usage: python train.py data_directory
parser = argparse.ArgumentParser(description='Optional add-ons.')
parser.add_argument('data_directory')
parser.add_argument('--save_dir', action='store')
parser.add_argument('--arch', action='store')
parser.add_argument('--learning_rate', action='store', type=float)
parser.add_argument('--hidden_units', action='store', type=int)
parser.add_argument('--epochs', action='store', type=int)
parser.add_argument('--gpu', action='store_true')
args = parser.parse_args()
# get category class mapping to names for flowers
with open('cat_to_name.json', 'r') as f:
cat_to_name = json.load(f)
# capture train and validation data and transform
train_data, valid_data = th.load_data(args.data_directory)
trainloader, validloader, class_to_idx = th.transform_data(
train_data, valid_data)
# Use GPU if it's available
device = torch.device('cuda') if (
args.gpu | torch.cuda.is_available()) else torch.device('cpu')
print('Building model...')
# set default to vgg11
args.arch = args.arch if args.arch else 'vgg11'
model = models.__dict__[args.arch](pretrained=True)
model.arch = args.arch
# update model classifier to customize for our problem (102 category probability calcs)
hidden_units = args.hidden_units if args.hidden_units else 4098
model.classifier = th.update_classifier(model, hidden_units)
print('Training model...')
epochs = args.epochs if args.epochs else 5
learning_rate = args.learning_rate if args.learning_rate else 0.001
th.train_model(model, trainloader, validloader, epochs, learning_rate,
device)
# add class_to_idx attribute to model
model.class_to_idx = class_to_idx
print('Saving model...')
save_dir = args.save_dir if args.save_dir else 'saved_models'
th.save_model(model, save_dir)
print('Trained model saved!')
def train(params):
# 1. 配置计算资源
config_gpus()
# 2. vocab
vocab = Vocab(params['vocab_path'], params['vocab_size'])
params['vocab_size'] = vocab.count
# 3. 构造模型
model = Seq2Seq(params, vocab) # 确保传入到模型的参数params的所有制不会再被修改，不然会报错。
# # 4. 模型存储
checkpoint = tf.train.Checkpoint(Seq2Seq=model)
checkpoint_manager = tf.train.CheckpointManager(
checkpoint,
directory=path.join(params['checkpoint_dir'], 'seq2seq_model'),
max_to_keep=5)
if params['restore']:
checkpoint.restore(checkpoint_manager.latest_checkpoint)
# 5. 模型训练
train_model(model, vocab, params, checkpoint_manager)
exit(0)
else:
device = 'cpu'
print("Further training will be done on cpu, switch to GPU\n")
print("Selected Device: ", device, "\n")
# load the datasets
data, loader = load_data(data_dir)
# make model
model = make_model(arch, hidden_units, drop)
model.to(device)
# set optimizer state according to arch
if model.name == 'vgg16' or model.name == 'densenet121':
optimizer = optim.Adam(model.classifier.parameters(), lr=lr)
elif model.name == 'resnet50':
optimizer = optim.Adam(model.fc.parameters(), lr=lr)
# train model, get new state of optimizer in order to save it in checkpoint
trained_model, optimizer = train_model(model, optimizer, epochs, device, data, loader)
# check accuracy
compute_accuracy(trained_model, loader, device)
# save model
save_checkpoint(model , arch, optimizer, lr, epochs, hidden_units, drop, save_dir)
help='criterio de solución')
parser.add_argument('--random_state',
type=int,
default=42,
help='Entero aleatorio')
parser.add_argument('--class_weight',
type=int,
default='balanced',
help='data balanceada')
args = parser.parse_args()
run = Run.get_context()
ws = run.experiment.workspace
datastore = ws.get_default_datastore()
input_ds = Dataset.get_by_name(ws, 'moa_ds')
data = input_ds.to_pandas_dataframe()
dataframe = validate_data(data)
X_train, X_test, y_train, y_test = split_data(dataframe)
model = train_model(X_train,
y_train,
save=True,
criterion=args.criterion,
random_state=args.random_state,
class_weight=args.class_weight)
y_pred = model.predict(X_test)
y_prob = model.predict_proba(X_test)
print(f"balanced_accuracy: {balanced_accuracy_score(y_test, y_pred)}")
print(f"log_loss: {log_loss(y_test, y_prob)}")
run.log('balanced_accuracy', balanced_accuracy_score(y_test, y_pred))
from sklearn.metrics import accuracy_score
from train_helper import validate_data, split_data, train_model
from sklearn.metrics import log_loss, balanced_accuracy_score
from azureml.core import Run
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('--data_path',
type=str,
default="./data/",
help='Path to the training data')
parser.add_argument('--file_name',
type=str,
default="dataset.csv",
help='Filename')
args = parser.parse_args()
print("===== DATA =====")
print("DATA PATH: " + args.data_path)
print("FILE NAME: " + args.file_name)
print("LIST FILES IN DATA PATH...")
print(os.listdir(args.data_path))
print("================")
data = pd.read_csv(args.data_path + args.file_name)
datos = validate_data(data)
X_train, y_train, X_test, y_test = split_data(datos)
model = train_model(X_train, y_train, save=True)
y_pred = model.predict(X_test)
y_prob = model.predict_proba(X_test)
print(f"balanced_accuracy: {balanced_accuracy_score(y_test, y_pred)}")
print(f"log_loss: {log_loss(y_test, y_prob)}")
#!/usr/bin/env python
import train_helper
n_epochs = 20
batch_size = 32
dataloaders, class_names = train_helper.load_data('../../data',
batch_size=batch_size)
model, criterion, optimizer, scheduler = train_helper.get_model(
dataloaders, n_epochs)
model = train_helper.train_model(model,
criterion,
optimizer,
scheduler,
dataloaders,
n_epochs=n_epochs)
train_helper.save_model(model, 'model.pt')
train_helper.save_mobile_model(model, 'model_mobile.pt')
torch.utils.data.DataLoader(image_datasets["train"],
batch_size=64,
shuffle=True),
"test":
torch.utils.data.DataLoader(image_datasets["test"], batch_size=64),
"valid":
torch.utils.data.DataLoader(image_datasets["valid"], batch_size=64)
}
if in_args.gpu:
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
if (device == "cpu"):
print("not found gpu,.... running on cpu")
else:
device = "cpu"
if (in_args.arch == "densenet"):
model = thelp.densenet121_FlowerModel(in_args.nhu)
else:
model = thelp.vgg11_FlowerModel(in_args.nhu)
criterion = nn.NLLLoss()
optimizer = optim.Adam(model.classifier.parameters(), lr=in_args.lr)
print(in_args)
thelp.train_model(model,
dataloaders,
criterion,
optimizer,
image_datasets,
device=device,
epochs=in_args.epochs,
nhu=in_args.nhu)
