def run_resnet2(t, v): # All layers frozen, except fc
tfms = transforms.Compose([
transforms.Resize((image_size, image_size)),
# transforms.CenterCrop(params['image_size']),
transforms.ToTensor(),
# Converts a PIL Image or numpy.ndarray (H x W x C) in the range [0, 255] to a torch.FloatTensor of shape (C x H x W) in the range [0.0, 1.0]
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
])
train_items, train_labels = t['id_aug'].values, t['label'].values
valid_items, valid_labels = v['id_aug'].values, v['label'].values
train_ds = ImageDataset(train_items, train_labels, tfms)
valid_ds = ImageDataset(valid_items, valid_labels, tfms)
net = Resnet(dropout=0.)
for layer in [
net.resnet.layer1, net.resnet.layer2, net.resnet.layer3,
net.resnet.layer4
]:
for param in layer.parameters():
param.requires_grad = False
print_model_details(net, params)
optimizer = Adam(net.parameters(), lr=lr)
criterion = nn.CrossEntropyLoss()
trainer = Trainer(train_ds=train_ds,
valid_ds=valid_ds,
model=net,
criterion=criterion,
optimizer=optimizer,
params=params,
callbacks=cbc,
metrics=mc)
trainer.train()
def test_train_method(self):
file_name = 'test/test_data/attention_test.txt'
fine_tune_model_name = '../models/glove_model_40.pth'
self.test_data_loader_attention = DataLoaderAttention(
file_name=file_name)
self.test_data_loader_attention.load_data()
source2index, index2source, target2index, index2target, train_data = \
self.test_data_loader_attention.load_data()
HIDDEN_SIZE = 512
NUM_LAYERS = 2
KERNEL_SIZE = 2
EMBEDDING_SIZE = 50
SOURCE_VOCAB_SIZE = len(source2index)
TARGET_VOCAB_SIZE = len(target2index)
qrnn = QRNNModel(QRNNLayer, NUM_LAYERS, KERNEL_SIZE, HIDDEN_SIZE,
EMBEDDING_SIZE, SOURCE_VOCAB_SIZE, TARGET_VOCAB_SIZE)
self.trainer = Trainer(epoch=100, fine_tune_model=fine_tune_model_name)
self.trainer.train_qrnn(
train_data=train_data,
source2index=source2index,
target2index=target2index,
index2source=index2source,
index2target=index2target,
qrnn_model=qrnn,
)
def test_train_method(self):
file_name = 'test/test_data/attention_test.txt'
fine_tune_model_name = '../models/glove_model_40.pth'
self.test_data_loader_attention = DataLoaderAttention(
file_name=file_name)
self.test_data_loader_attention.load_data()
source2index, index2source, target2index, index2target, train_data = \
self.test_data_loader_attention.load_data()
EMBEDDING_SIZE = 50
HIDDEN_SIZE = 32
encoder = Encoder(len(source2index), EMBEDDING_SIZE, HIDDEN_SIZE, 3,
True)
decoder = Decoder(len(target2index), EMBEDDING_SIZE, HIDDEN_SIZE * 2)
self.trainer = Trainer(fine_tune_model=fine_tune_model_name)
self.trainer.train_attention(
train_data=train_data,
source2index=source2index,
target2index=target2index,
index2source=index2source,
index2target=index2target,
encoder_model=encoder,
decoder_model=decoder,
)
def test_train_method(self):
self.test_data_loader = DataLoader()
self.test_japanese_wiki_data = 'test/test_data/jawiki_test.txt'
test_word2index, test_index2word, test_window_data, \
test_X_ik, test_weightinhg_dict = self.test_data_loader.load_data(file_name=self.test_japanese_wiki_data) # noqa
self.test_prepare_train_data = PrepareTrainData()
test_train_data = \
self.test_prepare_train_data.prepare_train_data_method(
window_data=test_window_data,
word2index=test_word2index,
weighting_dic=test_weightinhg_dict,
X_ik=test_X_ik)
self.model = Glove(vocab_size=len(test_word2index))
self.trainer = Trainer(model=self.model)
self.trainer.train_method(train_data=test_train_data)
word_similarity = self.trainer.word_similarity(
target=self.test_data_loader.vocab[0],
vocab=self.test_data_loader.vocab,
word2index=test_word2index,
top_rank=2)
word_similarity_check = ['<', '>', 's']
word_similarity_bool = False
for word in word_similarity:
if word[0] in word_similarity_check:
word_similarity_bool = True
assert word_similarity_bool is True
def main():
parser = argparse.ArgumentParser(description="Training attention model")
parser.add_argument(
"-t",
"--train_data",
metavar="train_data",
type=str,
default='../data/processed/source_replay_twitter_data.txt',
dest="train_data",
help="set the training data ")
parser.add_argument("-e",
"--embedding_size",
metavar="embedding_size",
type=int,
default=50,
dest="embedding_size",
help="set the embedding size ")
parser.add_argument("-H",
"--hidden_size",
metavar="hidden_size",
type=int,
default=512,
dest="hidden_size",
help="set the hidden size ")
parser.add_argument("-f",
"--fine_tune_model_name",
metavar="fine_tune_model_name",
type=str,
default='../models/glove_wiki/glove_model_40.pth',
dest="fine_tune_model_name",
help="set the fine tune model name ")
args = parser.parse_args()
data_loader_attention = DataLoaderAttention(file_name=args.train_data)
data_loader_attention.load_data()
source2index, index2source, target2index, index2target, train_data = \
data_loader_attention.load_data()
EMBEDDING_SIZE = args.embedding_size
HIDDEN_SIZE = args.hidden_size
encoder = Encoder(len(source2index), EMBEDDING_SIZE, HIDDEN_SIZE, 3, True)
decoder = Decoder(len(target2index), EMBEDDING_SIZE, HIDDEN_SIZE * 2)
trainer = Trainer(epoch=600,
batch_size=64,
fine_tune_model=args.fine_tune_model_name)
trainer.train_attention(train_data=train_data,
source2index=source2index,
target2index=target2index,
index2source=index2source,
index2target=index2target,
encoder_model=encoder,
decoder_model=decoder)
def train():
opt = TrainOptions().parse()
if opt.distributed:
init_dist()
opt.batchSize = opt.batchSize // len(opt.gpu_ids)
### setup dataset
data_loader = CreateDataLoader(opt)
dataset = data_loader.load_data()
pose = 'pose' in opt.dataset_mode
### setup trainer
trainer = Trainer(opt, data_loader)
### setup models
model, flowNet = create_model(opt, trainer.start_epoch)
flow_gt = conf_gt = [None] * 2
for epoch in range(trainer.start_epoch, opt.niter + opt.niter_decay + 1):
trainer.start_of_epoch(epoch, model, data_loader)
n_frames_total, n_frames_load = data_loader.dataset.n_frames_total, opt.n_frames_per_gpu
for idx, data in enumerate(dataset, start=trainer.epoch_iter):
trainer.start_of_iter()
if not opt.no_flow_gt:
data_list = [
data['tgt_label'], data['ref_label']
] if pose else [data['tgt_image'], data['ref_image']]
flow_gt, conf_gt = flowNet(data_list, epoch)
data_list = [
data['tgt_label'], data['tgt_image'], flow_gt, conf_gt
]
data_ref_list = [data['ref_label'], data['ref_image']]
data_prev = [None, None]
############## Forward Pass ######################
for t in range(0, n_frames_total, n_frames_load):
data_list_t = get_data_t(data_list, n_frames_load,
t) + data_ref_list + data_prev
g_losses, generated, data_prev = model(
data_list_t, save_images=trainer.save, mode='generator')
g_losses = loss_backward(opt, g_losses,
model.module.optimizer_G)
d_losses = model(data_list_t, mode='discriminator')
d_losses = loss_backward(opt, d_losses,
model.module.optimizer_D)
loss_dict = dict(
zip(model.module.lossCollector.loss_names,
g_losses + d_losses))
if trainer.end_of_iter(loss_dict,
generated + data_list + data_ref_list,
model):
break
trainer.end_of_epoch(model)
def test_train(self):
env = Env(balance=250000,
FX_DATA_FILE='../data/raw/FX_Demo/sample_USD_JPY_S5.pickle')
agent = Agent(input_data_shape=(10, ))
mount_agent = Agent(actions=10, input_data_shape=(10, ))
print(len(env.fx_time_data_buy))
trainer = Trainer(env,
agent,
mount_agent,
Adam(lr=1e-6),
data_end_index=len(env.fx_time_data_buy) - 2)
trainer.train()
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--config', type=str, default="config/base.yml")
args = parser.parse_args()
print("="*30 + "CONFIG INFOR" + "="*30)
config = Cfg.load_config_from_file(args.config)
print(config)
trainer = Trainer(config)
trainer.train()
def main():
#load logging
if constants.ENABLE_LOGGING == True:
load_logging()
else:
logging.getLogger().disabled = True
#load the config
CONFIG = load_config(constants.CONFIG_PATH)
#check if cuda is available & is enabled in config
constants.USE_CUDA = CONFIG['use_cuda'] & torch.cuda.is_available()
if constants.USE_CUDA == True:
constants.DEVICE = 'cuda:0'
else:
constants.DEVICE = 'cpu'
print(constants.DEVICE)
print(torch.cuda.is_available())
#get the data generators
vocab, train_generator, validation_generator = load_data(
constants.DATASET_PATH, CONFIG)
#use incremented vocabulary size because we have an extra character which isn't
#in the dictionary : 0 - padding character
model = MyModel(vocab_size=vocab.size() + 1,
embedding_size=CONFIG['embedding_size'],
rnn_size=CONFIG['rnn_size'],
output_size=CONFIG['output_size'])
model = model.to(constants.DEVICE)
trainer = Trainer(model=model,
vocab=vocab,
train_generator=train_generator,
val_generator=validation_generator,
epochs=CONFIG['epochs'],
batch_size=CONFIG['batch_size'],
max_grad_norm=CONFIG['max_grad_norm'],
lr=CONFIG['learning_rate'],
loss=CONFIG['loss'],
optim=CONFIG['optimizer'],
train_verbose=CONFIG['train_verbose'],
val_verbose=CONFIG['validation_verbose'])
trainer.train()
if CONFIG['save_model'] == True:
save_model(model)
def select(id):
sql = "SELECT * FROM Trainers WHERE id = %s"
values = [id]
sql_result = run_sql(sql, values)[0]
trainer = Trainer(sql_result['name'], sql_result['email'],
sql_result['phone'], sql_result['specialism'],
sql_result['id'])
return trainer
def update_trainer(id):
name = request.form["name"]
email = request.form["email"]
phone = request.form["phone"]
specialism = request.form["specialism"]
updated_trainer = Trainer(name, email, phone, specialism, id)
trainer_repository.update(updated_trainer)
return redirect("/trainers")
def select_all():
trainers = []
sql = 'SELECT * FROM trainers'
results = run_sql(sql)
for row in results:
trainer = Trainer(row['name'], row['active'], row['id'])
trainers.append(trainer)
return trainers
def create_trainer():
name = request.form["name"]
email = request.form["email"]
phone = request.form["phone"]
specialism = request.form["specialism"]
new_trainer = Trainer(name, email, phone, specialism)
trainer_repository.save(new_trainer)
return redirect("/trainers")
def trainer():
return Trainer(
name='Sebastian',
lastName='Herrera',
age=21,
phone=632432654,
email='*****@*****.**'
)
def select(id):
trainer = None
sql = "SELECT * FROM trainers WHERE id = %s"
values = [id]
result = run_sql(sql, values)[0]
if result is not None:
trainer = Trainer(result['name'], result['number'], result['id'])
return trainer
def main():
parser = argparse.ArgumentParser(description="Training glove model")
parser.add_argument(
"-c",
"--train_data",
metavar="train_data",
# type=str, default='../data/raw/jawiki_only_word_random_choose.txt',
type=str,
default='../data/raw/source_replay_twitter_data_sort.txt',
dest="train_data",
help="set the training data ")
parser.add_argument("-e",
"--embedding_size",
metavar="embedding_size",
type=int,
default=300,
dest="embedding_size",
help="set the embedding size")
args = parser.parse_args()
data_loader = DataLoader()
japanese_wiki_data = args.train_data
word2index, index2word, window_data, X_ik, weightinhg_dict = \
data_loader.load_data(file_name=japanese_wiki_data) # noqa
print(word2index)
prepare_train_data = PrepareTrainData()
train_data = \
prepare_train_data.prepare_train_data_method(
window_data=window_data,
word2index=word2index,
weighting_dic=weightinhg_dict,
X_ik=X_ik)
model = Glove(vocab_size=len(word2index),
projection_dim=args.embedding_size)
trainer = Trainer(model=model)
trainer.train_method(train_data=train_data)
word_similarity = trainer.word_similarity(target=data_loader.vocab[0],
vocab=data_loader.vocab,
word2index=word2index,
top_rank=2)
print(word_similarity)
def select(id):
trainer = None
sql = "SELECT * FROM trainers WHERE id = %s"
values = [id]
result = run_sql(sql, values)[0]
if result is not None:
location = Trainer(result['first_name'], result['last_name'], result['id'] )
return location
def select_all():
trainers = []
sql = "SELECT * FROM trainers"
results = run_sql(sql)
for sql_result in results:
trainer = Trainer(sql_result['name'], sql_result['email'],
sql_result['phone'], sql_result['specialism'],
sql_result['id'])
trainers.append(trainer)
return trainers
def select_all():
trainers = []
sql = "SELECT * FROM trainers"
results = run_sql(sql)
for row in results:
trainer = Trainer(row['first_name'], row['last_name'], row['id'])
trainers.append(trainer)
return trainers
def main(config):
# Load data loaders and model
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
loader_train, loader_test, loader_valid = get_dataloaders(
**config["dataloader"])
if config["use_pretrained_model"]:
model = build_pretrained_model(config["model_name"])
else:
model = MyCNN(im_size=config["dataloader"]["im_size"],
config_block=config["model"]["block"],
n_out_channels=config["model"]["n_out_channels"],
n_hidden_neurons=config["model"]["n_hidden_neurons"])
# Train model
trainer = Trainer(
model=model,
loader_train=loader_train,
loader_valid=loader_valid,
params_optim=config["params_optim"],
params_lr_scheduler=None, # config["params_lr_scheduler"],
device=device,
**config["trainer"])
trainer.train(n_iter=config["train"]["n_iter"])
# Test model on new observations (to see model's performance in a real life scenario)
conf_mat_test = get_confusion_matrix(
model=trainer.model,
threshold=config["trainer"]["threshold"],
data_loader=loader_test,
device=device)
f1_test = round(get_f1_score(conf_mat_test), 3)
acc_test = round(get_accuracy(conf_mat_test), 3)
print("On the test set, F1 score = {} and accuracy = {}".format(
f1_test, acc_test))
# Explain some predictions with SHAP values
e = Explainer(model, loader_train, device, savepath="./saved/figures")
images_test, label_test = sample_data(loader_test, 5)
e.fit()
e.explain(images_test, label_test)
def main():
parser = argparse.ArgumentParser(
description='Execute train reinforcement learning.')
parser.add_argument(
'--dataset_name',
type=str,
default="../data/raw/FX_Demo/sample10000_USD_JPY_S5.pickle",
help='an integer for the accumulator')
args = parser.parse_args()
print(args.dataset_name)
env = Env(balance=250000, FX_DATA_FILE=args.dataset_name)
agent = Agent(input_data_shape=(10, ))
mount_agent = Agent(actions=10, input_data_shape=(10, ))
trainer = Trainer(env,
agent,
mount_agent,
Adam(lr=1e-6),
data_end_index=len(env.fx_time_data_buy) - 2)
trainer.train()
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--config", default="./configs/" + CONFIG_FILE)
option = parser.parse_args()
config = json.load(open(option.config))
model_name = config["model"]
features = config["features"]
target_name = config["target_name"]
params = config["model_params"]
#lineNotify("leanrning start")
X, y, test = load_data(features, target_name)
#logger
get_logger(VERSION).info(test.shape)
get_logger(VERSION).info(option.config)
get_logger(VERSION).info(params)
get_logger(VERSION).info("====== CV score ======")
trainer = Trainer(model_name, params, features)
val_score, val_log = trainer.cross_validation(X,
y,
test,
random_state=RS,
n_split=K,
importance_f=imp,
task=TASK)
get_logger(VERSION).info(np.mean(val_score["lgb"]))
get_logger(VERSION).info(val_log)
# prediction
preds = trainer.predict()
#submit
trainer.create_submit(preds, val_score)
def test_model():
train_dataset = torch.load(
os.path.join(paths.experiment_data_dir, 'train.data'))
dev_dataset = torch.load(
os.path.join(paths.experiment_data_dir, 'dev.data'))
test_dataset = torch.load(
os.path.join(paths.experiment_data_dir, 'test.data'))
vocab = torch.load(paths.vocab_path)
model = RelationClassifier(encoder_type=args.encoder,
num_classes=len(train_dataset.label_map),
vocab_size=vocab.size(),
embed_dim=vocab.embed_dim,
model_config=ModelConfig,
drop_rate=args.drop,
use_cuda=args.cuda)
model.load_state_dict(torch.load(paths.best_model_path))
trainer = Trainer(model, use_cuda=args.cuda)
# train_loss, train_acc, train_f1 = trainer.eval(train_dataset, vocab, 'trainset')
# print('Train: loss {}, acc {}, f1 {}'.format(train_loss, train_acc, train_f1))
dev_acc = trainer.eval(dev_dataset, vocab, 'devset')
print('Dev: acc {}'.format(dev_acc))
test_acc = trainer.eval(test_dataset, vocab, 'testset')
print('Test: acc {}'.format(test_acc))
def training(ngrams_path, resume=False, checkpoint_path=""):
list_ngrams = load_dataset(ngrams_path)
trainer = Trainer(alphabets, list_ngram=list_ngrams)
# Resume training
if resume:
trainer.load_checkpoint(checkpoint_path)
# Training
trainer.train()
def main(_):
# creating the directories for model weights,
# logs and tensorboard logs
prepare_dirs(config)
rng = np.random.RandomState(config.random_seed)
tf.set_random_seed(config.random_seed)
read_semantics = config.read_semantics
if read_semantics and not config.semantic_regularizer:
model = DsNet(config)
print("using DS-net architecture!")
elif config.semantic_regularizer:
model = DepthModelRegularized(config)
print("using Depth-net regularized architecture!")
else:
model = DepthModel(config)
print("using depth-net architecture!")
# creating the dataset and the data loader objects
dataset = Dataset(config=config,
name="kitti",
read_semantics=read_semantics)
data_loader = DataLoader(dataset=dataset, shuffle_data=True)
# creating our trainer
trainer = Trainer(model=model,
data_loader=data_loader,
config=config,
rng=rng)
# training session
if config.is_train:
if config.resume_training:
trainer.resume_training()
else:
trainer.train()
# test session
else:
trainer.test(show_figure=False, save_figures=True, is_real_kitti=True)
def train_model():
train_dataset = torch.load(
os.path.join(paths.experiment_data_dir, 'train.data'))
dev_dataset = torch.load(
os.path.join(paths.experiment_data_dir, 'dev.data'))
test_dataset = torch.load(
os.path.join(paths.experiment_data_dir, 'test.data'))
vocab = torch.load(paths.vocab_path)
model = RelationClassifier(encoder_type=args.encoder,
num_classes=len(train_dataset.label_map),
vocab_size=vocab.size(),
embed_dim=vocab.embed_dim,
model_config=ModelConfig,
drop_rate=args.drop,
use_cuda=args.cuda,
attention=args.attention)
# init with pre-trained embeddings
model.argument_encoder.emb.weight.data.copy_(vocab.embeddings)
# model.argument_encoder.emb.weight.requires_grad = False
criterion = nn.CrossEntropyLoss()
params_to_train = filter(lambda p: p.requires_grad, model.parameters())
if args.cuda:
model.cuda(), criterion.cuda()
if args.optim == 'sgd':
optimizer = optim.SGD(params_to_train,
lr=args.lr,
weight_decay=args.wd)
elif args.optim == 'adam':
optimizer = optim.Adam(params_to_train,
lr=args.lr,
weight_decay=args.wd)
elif args.optim == 'adagrad':
optimizer = optim.Adagrad(params_to_train,
lr=args.lr,
weight_decay=args.wd)
elif args.optim == 'rprop':
optimizer = optim.Rprop(params_to_train, lr=args.lr)
trainer = Trainer(model, criterion, optimizer, args.cuda)
best_dev_acc = 0
for epoch in range(args.epochs):
print()
trainer.train(train_dataset, vocab, batch_size=args.batch_size)
# train_loss, train_acc, train_f1 = trainer.eval(train_dataset, vocab, 'trainset')
# print('Train: loss {}, acc {}, f1 {}'.format(train_loss, train_acc, train_f1))
dev_acc = trainer.eval(dev_dataset, vocab, 'devset')
print('Dev: acc {}'.format(dev_acc))
test_acc = trainer.eval(test_dataset, vocab, 'testset')
print('Test: acc {}'.format(test_acc))
if dev_acc > best_dev_acc:
best_dev_acc = dev_acc
print('Model saved to {}'.format(paths.best_model_path))
torch.save(model.state_dict(), paths.best_model_path)
from models.nurse import Nurse
from models.pokemon import Pokemon
from models.trainer import Trainer
import repositories.nurse_repository as nurse_repository
import repositories.pokemon_repository as pokemon_repository
import repositories.trainer_repository as trainer_repository
# pokemon_repository.delete_all()
# nurse_repository.delete_all()
# trainer_repository.delete_all()
nurse_1 = Nurse("Joy", "Fire")
nurse_repository.save(nurse_1)
trainer = Trainer("Simon", 12345)
trainer_repository.save(trainer)
pokemon = Pokemon('Seed Backman', 'Bulbasaur', 'Grass', '25/09/1990', trainer , 'burned')
pokemon.assign_nurse(nurse_1)
pokemon_repository.save(pokemon)
trainer.name = 'Simondovich'
trainer_repository.update(trainer)
nurse_1.name = "Betsy"
sequenceSize=settings.sequenceSize,
stepSize=settings.stepSize,
targetType="float",
sensors=settings.sensors)
print("Dataset read in time", time.time() - dsTime)
print("Trainset size:", len(trainingSet), "Testset size:", len(testSet))
print("Trainging on", [x[2] for x in trainingSet])
print("Testing on", [x[2] for x in testSet])
module = SingleGRU()
if settings.loadModel:
module.load_state_dict(torch.load(settings.modelPath))
trainer = Trainer(module=module,
optimizer=module.optimizer,
learningRate=module.lr,
lossFunction=module.lossFunction)
if not settings.loadModel:
trainer.train(trainingSet, testSet, module.numEpochs)
torch.save(module.state_dict(), module.modelPath)
print("remember to rename model file if wanted")
trainer.printBenchmarks(trainingSet + testSet)
trainer.storeBenchmarks("benchmarks" + module.modelPath[:-3] + ".txt")
print("possible scenarios to store:")
scenarios = [x[2] for x in (trainingSet + testSet)]
print(scenarios)
scen = input("which scenario should be stored?")
while ("Scenario-" + scen) not in scenarios:
scen = input("which scenario should be stored?")
from models.trainer import Trainer
config = {"agent": "A3C", "episodes": 100}
if __name__ == "__main__":
trainer = Trainer(config)
trainer.train()
seq2seq = Seq2seq(config, len(src.vocab), len(tgt.vocab), tgt.sos_id, tgt.eos_id,
pretrained_pos_weight)
else :
seq2seq = Seq2seq(config, len(src.vocab), len(tgt.vocab), tgt.sos_id, tgt.eos_id)
if torch.cuda.is_available():
seq2seq.cuda()
for param in seq2seq.parameters():
param.data.uniform_(-0.08, 0.08)
# train
t = Trainer(loss=loss, batch_size=128,
learning_rate=0.001,
checkpoint_every=50,
print_every=100,
hidden_size=config["hidden_size"],
path=save_path,
file_name=config["rnn_cell"] + "_" + str(i))
seq2seq, ave_loss, character_accuracy_list, sentence_accuracy_list, f1_score_list = t.train(seq2seq, train,
num_epochs=epochs, dev_data=dev,
optimizer=optimizer,
teacher_forcing_ratio=0.5)
character_accuracy.append(character_accuracy_list)
sentence_accuracy.append(sentence_accuracy_list)
f1_score.append(f1_score_list)
best_f1_score.append(max(f1_score_list))
# svae plot
