def main():
num_outputs = 10
architecture = ((2, 64), (2, 128), (4, 256), (4, 512), (4, 512))
net = nn.Sequential()
with net.name_scope():
net.add(
vgg_stack(architecture),
nn.Flatten(),
nn.Dense(4096, activation="relu"),
nn.Dropout(.5),
nn.Dense(4096, activation="relu"),
nn.Dropout(.5),
nn.Dense(num_outputs))
train_data, test_data = utils.load_data_fashion_mnist(
batch_size=64, resize=96)
ctx = utils.try_gpu()
net.initialize(ctx=ctx, init=init.Xavier())
loss = gluon.loss.SoftmaxCrossEntropyLoss()
trainer = gluon.Trainer(net.collect_params(), 'sgd', {
'learning_rate': 0.05})
utils.train(train_data, test_data, net, loss, trainer, ctx, num_epochs=100)
def train(net, ctx, train_data, test_data, batch_size=64, epochs=1, learning_rate=0.001, wd=0.001):
# 确保net的初始化在ctx上
net.collect_params().reset_ctx(ctx)
net.hybridize()
loss = gluon.loss.SoftmaxCrossEntropyLoss()
# 训练
trainer = gluon.Trainer(net.collect_params(), 'sgd', {
'learning_rate': learning_rate, 'wd': wd})
utils.train(train_data, test_data, net, loss, trainer, ctx, epochs, 10)
def main():
net = get_net()
net_str = '%s'%net
logging.info('ok')
logging.info(net_str)
if pretrained is not None:
net.load_params(pretrained,ctx=utils.try_gpu())
trainIter, testIter = get_train_test()
trainer,loss = get_trainer(net)
utils.train(trainIter, testIter, net, loss, trainer, utils.try_gpu(), 1000,print_batches=100, cpdir='cp')
def main():
net = get_net()
net_str = '%s'%net
#logging.info('ok')
logging.info(net_str)
if pretrained is not None:
net.load_params(pretrained,ctx=utils.try_gpu())
train_data, test_data = get_train_test()
trainer,loss = get_trainer(net)
utils.train(train_data, test_data, trainBatchSize,\
net, loss, trainer, utils.try_gpu(), 1000,\
500,0.1,print_batches=100, chk_pts_dir=checkpoints)
def main():
args = make_args()
config = configparser.ConfigParser()
utils.load_config(config, args.config)
for cmd in args.modify:
utils.modify_config(config, cmd)
with open(os.path.expanduser(os.path.expandvars(args.logging)), 'r') as f:
logging.config.dictConfig(yaml.load(f))
if args.run is None:
buffer = io.StringIO()
config.write(buffer)
args.run = hashlib.md5(buffer.getvalue().encode()).hexdigest()
logging.info('cd ' + os.getcwd() + ' && ' + subprocess.list2cmdline([sys.executable] + sys.argv))
train = Train(args, config)
train()
logging.info(pybenchmark.stats)
raise ValueError('Task should be ent, sim, or sentiment.')
model = None
print sys.argv
if params.nntype == 'lstm':
model = lstm_model_sim(We, params)
elif params.nntype == 'word':
model = word_model_sim(We, params)
elif params.nntype == 'proj':
model = proj_model_sim(We, params)
elif params.nntype == 'dan':
model = dan_model_sim(We, params)
elif params.nntype == 'rnn':
model = rnn_model_sim(We, params)
elif params.nntype == 'lstm_sentiment':
model = lstm_model_sentiment(We, params)
elif params.nntype == 'word_sentiment':
model = word_model_sentiment(We, params)
elif params.nntype == 'proj_sentiment':
model = proj_model_sentiment(We, params)
elif params.nntype == 'dan_sentiment':
model = dan_model_sentiment(We, params)
elif params.nntype == 'rnn_sentiment':
model = rnn_model_sentiment(We, params)
else:
"Error no type specified"
utils.train(model, train_data, params.devdata, params.testdata, params.traindata, words, params)
forward = fw_init.transduce(embedded)
backward = bw_init.transduce(embedded)
return dy.concatenate([forward[-1], backward[-1]])
def calc_lm_loss(words):
dy.renew_cg()
sm_w = dy.parameter(softmax_w)
sm_b = dy.parameter(softmax_b)
w_init = word_lstm.initial_state()
fw_init = char_fw_lstm.initial_state()
bw_init = char_bw_lstm.initial_state()
word_embeddings = [word_rep(w, fw_init, bw_init) for w in words]
labels = [word_vocab.get(w) for w in words]
s = w_init.add_input(word_rep(words[-1], fw_init, bw_init))
losses = []
for word_emb, label in zip(word_embeddings, labels):
logits = sm_w * s.output() + sm_b
loss = dy.pickneglogsoftmax(logits, label)
losses.append(loss)
s = s.add_input(word_emb)
return dy.esum(losses)
train(model, trainer, calc_lm_loss, num_epochs, "models/compositional")
from mxnet.gluon import nn
net = nn.Sequential()
with net.name_scope():
net.add(nn.Conv2D(channels=20, kernel_size=5, activation='relu'),
nn.MaxPool2D(pool_size=2, strides=2),
nn.Conv2D(channels=50, kernel_size=3, activation='relu'),
nn.MaxPool2D(pool_size=2, strides=2), nn.Flatten(),
nn.Dense(128, activation="relu"), nn.Dense(10))
from mxnet import gluon
import sys
sys.path.append('..')
import utils
ctx = utils.try_gpu()
net.initialize(ctx=ctx)
print('initialize weight on', ctx)
batch_size = 256
train_data, test_data = utils.load_data_fashion_mnist(batch_size)
loss = gluon.loss.SoftmaxCrossEntropyLoss()
trainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': 0.5})
utils.train(train_data, test_data, net, loss, trainer, ctx, num_epochs=5)
def main():
parser = argparse.ArgumentParser(description='PyTorch CIFAR Training')
parser.add_argument('--teacher_depth', default=100, type=int, help='')
parser.add_argument('--teacher_width', default=12, type=int, help='')
parser.add_argument('--teacher_mult', default=4.0, type=float, help='')
parser.add_argument('--depth', default=100, type=int, help='')
parser.add_argument('--width', default=11, type=int, help='')
parser.add_argument('--mult', default=4.0, type=float, help='')
parser.add_argument('--hkd', default=0., type=float, help='')
parser.add_argument('--temp', default=0., type=float, help='')
parser.add_argument('--gkd', default=0., type=float, help='')
parser.add_argument('--p', default=1, type=int, help='')
parser.add_argument('--seed', default=0, type=int, help='')
parser.add_argument('--k', default=128, type=int, help='')
parser.add_argument('--pool3', action='store_true', help='')
parser.add_argument('--intra_only', action='store_true', help='')
parser.add_argument('--inter_only', action='store_true', help='')
args = parser.parse_args()
save = "L2Loss_{}-{}-{}_teaches_{}-{}-{}_{}_{}_{}_{}_{}_{}_{}_{}_{}".format(
args.teacher_depth, args.teacher_width, args.teacher_mult, args.depth,
args.width, args.mult, args.hkd, args.temp, args.gkd, args.p, args.k,
args.pool3, args.intra_only, args.inter_only, args.seed)
device = 'cuda' if torch.cuda.is_available() else 'cpu'
trainloader, testloader, clean_trainloader = load_data(128)
file = "checkpoint/FinetuneDensenet{}-{}-{}_3.pth".format(
args.teacher_depth, args.teacher_width, args.teacher_mult)
teacher = torch.load(file)["net"].module
teacher = teacher.to(device)
# net = models.nearmobile.NearMobileNetStart(depth=args.depth,width=args.width,expansion=args.mult,num_classes=10)
net = models.densenet.densenet_cifar(n=args.depth,
growth_rate=args.width,
bottle_size=args.mult,
num_classes=10)
net = net.to(device)
# net.bn = teacher.bn
# net.linear = teacher.linear
# net.conv1 = teacher.conv1
# net.trans1 = copy.deepcopy(teacher.trans1)
# net.trans2 = copy.deepcopy(teacher.trans2)
# net.trans3 = copy.deepcopy(teacher.trans3)
parameters = list()
# ignore = [net.bn, net.linear, net.conv1]
ignore = list()
for a in net.parameters():
for b in ignore:
if a is b.weight or a is b.bias:
print(a.size())
continue
parameters.append(a)
if device == 'cuda':
net = torch.nn.DataParallel(net)
cudnn.benchmark = True
# optimizer = optim.Adam(parameters, lr=0.001)
# optimizer = optim.SGD(parameters, lr=0.1, momentum=0.9,nesterov=True,weight_decay=1e-4)
# scheduler = ReduceLROnPlateau(optimizer, factor=0.2,patience=5,threshold=1e-4,verbose=True)
optimizer = optim.SGD(net.parameters(),
lr=0.05,
momentum=0.9,
nesterov=True,
weight_decay=1e-4)
scheduler = MultiStepLR(optimizer, milestones=[150, 225], gamma=0.1)
predicted_teacher, labels = test(teacher,
clean_trainloader,
device,
save_name="no")
predicted_teacher, labels = test(teacher,
trainloader,
device,
save_name="no")
predicted_teacher, labels = test(teacher,
testloader,
device,
save_name="no")
for epoch in range(300):
print('Epoch: %d' % epoch)
train(net,
trainloader,
scheduler,
device,
optimizer,
teacher=teacher,
lambda_hkd=args.hkd,
lambda_gkd=args.gkd,
temp=args.temp,
classes=10,
power=args.p,
pool3_only=args.pool3,
k=args.k,
intra_only=args.intra_only,
inter_only=args.inter_only)
predicted_student, labels = test(net,
testloader,
device,
save_name=save)
consistency = 100 * (predicted_teacher == predicted_student).float()
teacher_true = predicted_teacher == labels
teacher_false = predicted_teacher != labels
consistency_true = consistency[teacher_true].mean()
consistency_false = consistency[teacher_false].mean()
consistency = consistency.mean()
print("Consistency: {:.2f} (T: {:.2f}, F: {:.2f})%".format(
consistency, consistency_true, consistency_false))
lr = optimizer.param_groups[0]['lr']
if lr <= 0.000001:
break
optimizer = optim.SGD(clf.parameters(), lr=args.lr, weight_decay=args.wd, momentum=args.mom)
criterion = nn.CrossEntropyLoss()
# Multiplies the LR with 0.1 at epoch 100 and 150 as mentioned in the paper
lmd = lambda x: 0.1 if x in [100,150] else 1
scheduler = lr_scheduler.MultiplicativeLR(optimizer, lr_lambda=lmd)
trainloader = DataLoader(fmnist_train, batch_size=args.batch_size, shuffle=True)
testloader = DataLoader(fmnist_test, batch_size=args.batch_size, shuffle=False)
best_loss = np.inf
for epoch in range(args.epochs):
t_loss, t_acc = train(epoch, trainloader, clf, criterion, optimizer, scheduler=None, msda=args.msda)
print('Epoch {}/{} (train) || Loss: {:.4f} Acc: {:.4f} LR: {(optimizer.param_groups[0]['lr']):.5f}'.format(epoch+1, EPOCHS, t_loss, t_acc, lr))
test_loss, test_acc = test(epoch, testloader, clf, criterion)
print('Epoch {}/{} (test) || Loss: {:.4f} Acc: {:.4f}'.format(epoch+1, EPOCHS, test_loss, test_acc))
scheduler.step()
if test_loss<best_loss:
torch.save(clf.state_dict(), os.path.join(args.save_dir, f'{args.msda}_weight.pt'))
best_loss = test_loss
parser.add_argument("--gpu", type=str, default="0", help="set gpu to '' to use CPU mode")
parser.add_argument("--num_epochs", type=int, default=45)
parser.add_argument("--embedding_dim", type=int, default=300, help="Word2Vec dimension")
parser.add_argument("--sequence_length", type=int, default=20, help="final length of each sequence (premise and hypothesis), padded with null-words if needed")
parser.add_argument("--num_units", type=int, default=100, help="LSTM output dimension (k in the original paper)")
args = parser.parse_args()
# PARAMETERS
parameters = {
"runs_dir": RUNS_DIR,
"embedding_dim": args.embedding_dim,
"num_units": args.num_units,
"num_epochs": args.num_epochs,
"learning_rate": args.learning_rate,
"keep_prob": args.keep_prob,
"model_name": args.model_name,
"gpu": args.gpu or None,
"batch_size": {"train": args.batch_size_train, "dev": args.batch_size_dev, "test": args.batch_size_test},
"sequence_length": args.sequence_length,
"weight_decay": args.weight_decay,
}
for key, parameter in parameters.iteritems():
print "{}: {}".format(key, parameter)
# MAIN
word2vec, dataset = load_data(data_dir=args.data_dir, word2vec_path=args.word2vec_path)
if args.train:
train(word2vec=word2vec, dataset=dataset, parameters=parameters)
# Then select only the probabilities corresponding to sampled actions
probabilities = actor(states)
probabilities = probabilities[range(states.shape[0]),
actions.flatten()]
loss_actor = (-torch.log(probabilities) * advantage).sum()
# Take the weighted average (helps convergence)
loss += loss_critic + loss_actor
total_len += states.shape[0]
loss = loss / total_len
optim_actor.zero_grad()
optim_critic.zero_grad()
loss.backward()
optim_actor.step()
optim_critic.step()
actor.eval()
critic.eval()
# Train, provide an env, function to get an action from state, and training function that accepts rollouts
train(env,
get_action,
train_epoch,
epochs=2000,
num_rollouts=10,
print_frequency=10,
plot_frequency=50)
from utils import train
import data
from resnet import resnet
import torch
from torch import nn
DEBUG = False
BATCH_SIZE = 64
LR = 1e-3
if __name__ == "__main__":
trainset, testset = data.get_train_test_set()
trainloader = torch.utils.data.DataLoader(trainset,
batch_size=BATCH_SIZE,
shuffle=True)
testloader = torch.utils.data.DataLoader(testset,
batch_size=BATCH_SIZE,
shuffle=False)
net = resnet(in_channel=3, num_classes=1)
optimizer = torch.optim.Adam(net.parameters(), lr=LR)
criterion = nn.BCELoss()
train(
net,
trainloader,
testloader,
20,
optimizer,
criterion,
debug=DEBUG,
)
if args.load_model:
model.load_state_dict(torch.load(args.model_filename))
# Move model to HammerBlade if using HB
if args.hammerblade:
model.to(torch.device("hammerblade"))
print(model)
# Quit here if dry run
if args.dry:
exit(0)
# Training
if args.training:
utils.train(model, trainloader, optimizer, loss_func, args)
# Inference
if args.inference:
num_correct = [0]
def collector(outputs, targets):
pred = outputs.cpu().max(1)[1]
num_correct[0] += pred.eq(targets.cpu().view_as(pred)).sum().item()
utils.inference(model, testloader, loss_func, collector, args)
num_correct = num_correct[0]
test_accuracy = 100. * (num_correct / len(testloader.dataset))
def main():
parser = argparse.ArgumentParser(
description='train a convolutional network' + 'on the MAPS dataset')
parser.add_argument('splits', type=str, help='on which splits to train')
parser.add_argument('run_path', type=str, help='where to write run state')
parser.add_argument(
'model',
choices=models.get_model_classes(),
help='any classname of model as defined in "models.py"')
parser.add_argument('--device', type=str, default='cuda')
args = parser.parse_args()
cuda = False
if args.device.startswith('cpu'):
cuda = False
elif args.device.startswith('cuda'):
cuda = True
else:
print('unknown device type "{}"'.format(args.device))
exit(-1)
run_path = args.run_path
if os.path.exists(run_path):
print('run_path "{}" already exists!'.format(run_path))
exit(-1)
##########################################
# prepare train data
train_filenames = filenames_from_splitfile(
os.path.join(args.splits, 'train'))
train_sequences = datasets.get_sequences(train_filenames)
train_dataset = ConcatDataset(train_sequences)
batch_size = 128
n_epochs = 500
# go with the original definition of an 'epoch' (aka the whole dataset ...)
# as opposed to 'arbitrary number of steps until we validate'
# n_steps = 8192
n_steps = len(train_dataset) // batch_size
print('adjusted n_steps', n_steps)
# one train loader for all train sequences
train_loader = DataLoader(train_dataset,
batch_size=batch_size,
sampler=ChunkedRandomSampler(
train_dataset, batch_size * n_steps),
num_workers=1,
pin_memory=True,
drop_last=True)
# prepare valid data as individual sequences
valid_filenames = filenames_from_splitfile(
os.path.join(args.splits, 'valid'))
# to get all of the sequence data, just write this:
valid_sequences = datasets.get_sequences(valid_filenames)
valid_loaders = []
for valid_sequence in valid_sequences:
valid_loader = DataLoader(valid_sequence,
batch_size=1024,
sampler=SequentialSampler(valid_sequence),
num_workers=0,
pin_memory=False,
drop_last=False)
valid_loaders.append(valid_loader)
log_dir = os.path.join(run_path, 'tensorboard')
ensure_empty_directory_exists(log_dir)
logger = SummaryWriter(log_dir=log_dir)
net_class = getattr(models, args.model, None)
if net_class is None:
raise RuntimeError(
'could not find model class named "{}" in "models.py"'.format(
args.model))
net = net_class()
if args.model == 'AllConv2016':
print('choosing AllConv2016 learnrate and learnrate schedule!')
# this does not train all that well ... validation loss stays high all the time?
optimizer = optim.SGD(net.parameters(),
lr=1.0,
momentum=0.9,
weight_decay=1e-5,
nesterov=False)
milestones = list(range(10, n_epochs, 10))
scheduler = optim.lr_scheduler.MultiStepLR(optimizer,
milestones,
gamma=0.5)
else:
print('choosing VGGNet2016 learnrate and learnrate schedule!')
# this leads to the results from 2016 in ~5 epochs
optimizer = optim.SGD(net.parameters(),
lr=0.1,
momentum=0.9,
weight_decay=1e-5,
nesterov=False)
scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer,
mode='min',
factor=0.5,
patience=20,
verbose=True,
min_lr=1e-4)
if cuda:
net.cuda()
train(cuda=cuda,
run_path=run_path,
net=net,
optimizer=optimizer,
scheduler=scheduler,
n_epochs=n_epochs,
train_loader=train_loader,
valid_loader=valid_loaders,
logger=logger)
def main():
torch.cuda.manual_seed(seed)
cudnn.benchmark = CUDNN
# model
model = tiramisu.FCDenseNet67(in_channels=8, n_classes=N_CLASSES)
model = model.cuda()
# model = torch.nn.DataParallel(model).cuda()
print(' + Number of params: {}'.format(
sum([p.data.nelement() for p in model.parameters()])))
model.apply(utils.weights_init)
optimizer = optim.RMSprop(model.parameters(),
lr=LEARNING_RATE,
weight_decay=WEIGHT_DECAY,
eps=1e-12)
criterion = nn.NLLLoss2d().cuda()
exp = experiment.Experiment(EXPNAME, EXPERIMENT)
exp.init()
START_EPOCH = exp.epoch
END_EPOCH = START_EPOCH + N_EPOCHS
# for epoch in range(1):
for epoch in range(START_EPOCH, END_EPOCH):
since = time.time()
# ### Collect data ###
# delete existing folder and old data
if os.path.exists(res_root_path):
shutil.rmtree(res_root_path)
utils.collect_data(ori_train_base_rp, res_train_base_rp)
utils.collect_data(ori_val_base_rp, res_val_base_rp)
# data loader
train_loader, val_loader = utils.data_loader(res_root_path)
### Train ###
trn_loss, trn_err = utils.train(model, train_loader, optimizer,
criterion, epoch)
print('Epoch {:d}: Train - Loss: {:.4f}\tErr: {:.4f}'.format(
epoch, trn_loss, trn_err))
time_elapsed = time.time() - since
print('Train Time {:.0f}m {:.0f}s'.format(time_elapsed // 60,
time_elapsed % 60))
### val ###
val_loss, val_err = utils.test(model, val_loader, criterion, epoch)
print('Val - Loss: {:.4f}, Error: {:.4f}'.format(val_loss, val_err))
time_elapsed = time.time() - since
print('Total Time {:.0f}m {:.0f}s\n'.format(time_elapsed // 60,
time_elapsed % 60))
### Save Metrics ###
exp.save_history('train', trn_loss, trn_err)
exp.save_history('val', val_loss, val_err)
### Checkpoint ###
exp.save_weights(model, trn_loss, val_loss, trn_err, val_err)
exp.save_optimizer(optimizer, val_loss)
## Early Stopping ##
if (epoch - exp.best_val_loss_epoch) > MAX_PATIENCE:
print(("Early stopping at epoch %d since no " +
"better loss found since epoch %.3").format(
epoch, exp.best_val_loss))
break
utils.adjust_learning_rate(LEARNING_RATE, LR_DECAY, optimizer, epoch,
DECAY_LR_EVERY_N_EPOCHS)
exp.epoch += 1
"""Calculate the head prediction accuracy of the model on the given dataset
of (encoded) input/output pairs."""
correct, total = 0, 0
for x, y in data:
pred_y = torch.argmax(model(x)[1], dim=1)
correct += (y[1] == pred_y).sum()
total += len(y[1])
return float(correct) / total
def loss(pred: Tuple[Tensor, Tensor], gold: Tuple[Tensor, Tensor]) -> Tensor:
criterion = nn.CrossEntropyLoss()
return criterion(pred[0], gold[0]) + criterion(pred[1], gold[1])
train(model,
enc_train,
enc_dev,
loss,
pos_accuracy,
epoch_num=10,
learning_rate=0.001,
report_rate=1)
train(model,
enc_train,
enc_dev,
loss,
dep_accuracy,
epoch_num=10,
learning_rate=0.0001,
report_rate=1)
else:
raise AssertionError("Wrong Model Type!")
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters())
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model.to(device)
criterion.to(device)
best_val_acc = 0.0
train_losses, train_accuracies = [], []
valid_losses, valid_accuracies = [], []
for epoch in range(NUM_EPOCHS):
train_loss, train_accuracy = train(model, device, train_loader, criterion, optimizer, epoch)
valid_loss, valid_accuracy, valid_results = evaluate(model, device, valid_loader, criterion)
train_losses.append(train_loss)
valid_losses.append(valid_loss)
train_accuracies.append(train_accuracy)
valid_accuracies.append(valid_accuracy)
is_best = valid_accuracy > best_val_acc # let's keep the model that has the best accuracy, but you can also use another metric.
if is_best:
best_val_acc = valid_accuracy
torch.save(model, os.path.join(PATH_OUTPUT, save_file))
plot_learning_curves(train_losses, valid_losses, train_accuracies, valid_accuracies)
momentum=0.9).minimize(loss)
# merged_summary_op = tf.summary.merge_all()
# weights_conv1 = utils.get_weight_variable('conv1')
# weights_conv2 = utils.get_weight_variable('conv2')
with utils.create_session() as sess:
saver = tf.train.Saver()
sess.run(tf.global_variables_initializer())
best_test_accuracy = -1
best_test_accuracy_epoch = 0
for epoch in range(max_epochs):
train_accuracy, train_loss, train_time_spent = utils.train(
data.train, train_batch_size,
lambda batch_x, batch_y: sess.run([optimazer, guesses, loss],
feed_dict={
x: batch_x,
y_true: batch_y
}))
test_accuracy, test_time_spent = utils.test(
data.test, test_batch_size,
lambda batch_x, batch_y: sess.run([guesses],
feed_dict={
x: batch_x,
y_true: batch_y
}))
print(
"epoch: {:02d} train_accuracy: {:3.2f}% train_loss: {:2.3f} time: {:2.2f}s\n"
" test_accuracy: {:3.2f}% time: {:2.2f}s"
.format(epoch + 1, train_accuracy, train_loss, train_time_spent,
def main():
parser = argparse.ArgumentParser()
arg = parser.add_argument
arg('root', help='checkpoint root')
arg('--batch-size', type=int, default=32)
arg('--patch-size', type=int, default=256)
arg('--n-epochs', type=int, default=100)
arg('--lr', type=float, default=0.0001)
arg('--workers', type=int, default=2)
arg('--fold', type=int, default=1)
arg('--bg-weight', type=float, default=1.0, help='background weight')
arg('--dice-weight', type=float, default=0.0)
arg('--n-folds', type=int, default=5)
arg('--stratified', action='store_true')
arg('--mode', choices=[
'train', 'valid', 'predict_valid', 'predict_test', 'predict_all_valid'],
default='train')
arg('--model-path',
help='path to model file to use for validation/prediction')
arg('--clean', action='store_true')
arg('--epoch-size', type=int)
arg('--limit', type=int, help='Use only N images for train/valid')
arg('--min-scale', type=float, default=1)
arg('--max-scale', type=float, default=1)
arg('--test-scale', type=float, default=0.5)
arg('--oversample', type=float, default=0.0,
help='sample near lion with given probability')
arg('--with-head', action='store_true')
arg('--pred-oddity', type=int, help='set to 0/1 to predict even/odd images')
args = parser.parse_args()
coords = utils.load_coords()
train_paths, valid_paths = utils.train_valid_split(args, coords)
root = Path(args.root)
model = UNetWithHead() if args.with_head else UNet()
model = utils.cuda(model)
criterion = Loss(dice_weight=args.dice_weight, bg_weight=args.bg_weight)
loader_kwargs = dict(
min_scale=args.min_scale, max_scale=args.max_scale,
downscale=args.with_head,
)
if args.mode == 'train':
train_loader, valid_loader = (
utils.make_loader(
SegmentationDataset, args, train_paths, coords,
oversample=args.oversample, **loader_kwargs),
utils.make_loader(
SegmentationDataset, args, valid_paths, coords,
deterministic=True, **loader_kwargs))
if root.exists() and args.clean:
shutil.rmtree(str(root))
root.mkdir(exist_ok=True)
root.joinpath('params.json').write_text(
json.dumps(vars(args), indent=True, sort_keys=True))
utils.train(args, model, criterion,
train_loader=train_loader, valid_loader=valid_loader,
save_predictions=save_predictions)
elif args.mode == 'valid':
utils.load_best_model(model, root, args.model_path)
valid_loader = utils.make_loader(
SegmentationDataset, args, valid_paths, coords,
deterministic=True, **loader_kwargs)
utils.validation(model, criterion,
tqdm.tqdm(valid_loader, desc='Validation'))
else:
utils.load_best_model(model, root, args.model_path)
if args.mode in {'predict_valid', 'predict_all_valid'}:
if args.mode == 'predict_all_valid':
# include all paths we did not train on (makes sense only with --limit)
valid_paths = list(
set(valid_paths) | (set(utils.labeled_paths()) - set(train_paths)))
predict(model, valid_paths, out_path=root,
patch_size=args.patch_size, batch_size=args.batch_size,
min_scale=args.min_scale, max_scale=args.max_scale,
downsampled=args.with_head)
elif args.mode == 'predict_test':
out_path = root.joinpath('test')
out_path.mkdir(exist_ok=True)
predicted = {p.stem.split('-')[0] for p in out_path.glob('*.npy')}
test_paths = [p for p in utils.DATA_ROOT.joinpath('Test').glob('*.jpg')
if p.stem not in predicted]
if args.pred_oddity is not None:
assert args.pred_oddity in {0, 1}
test_paths = [p for p in test_paths
if int(p.stem) % 2 == args.pred_oddity]
predict(model, test_paths, out_path,
patch_size=args.patch_size, batch_size=args.batch_size,
test_scale=args.test_scale,
is_test=True, downsampled=args.with_head)
else:
parser.error('Unexpected mode {}'.format(args.mode))
train_set = CIFAR10('C:/DATASETS/cifar10/', train=True, transform=train_tf)
train_data = torch.utils.data.DataLoader(train_set,
batch_size=64,
shuffle=True)
test_set = CIFAR10('C:/DATASETS/cifar10/', train=False, transform=test_tf)
test_data = torch.utils.data.DataLoader(test_set,
batch_size=128,
shuffle=False)
net = resnet(3, 10)
optimizer = torch.optim.SGD(net.parameters(), lr=0.01)
criterion = nn.CrossEntropyLoss()
#%%
train(net, train_data, test_data, 10, optimizer, criterion)
#%%
# 不使用数据增强
def data_tf(x):
im_aug = tfs.Compose([
tfs.Resize(96),
tfs.ToTensor(),
tfs.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])
])
x = im_aug(x)
return x
train_set = CIFAR10('C:/DATASETS/cifar10/', train=True, transform=data_tf)
params.clip = None
else:
params.clip = None
model = None
print sys.argv
if params.domain == "pos":
trainexamples = utils.get_pos_data(params.traindata)
devexamples = utils.get_pos_data(params.devdata)
testexamples = utils.get_pos_data(params.testdata)
model = pos_tagging(params, trainexamples)
model.train(params, trainexamples, devexamples, testexamples)
else:
examples = utils.get_ppdb_data(params.train)
if args.num_examples:
examples = examples[0:args.num_examples]
if params.nntype == 'charlstm':
model = char_lstm_model(params)
elif params.nntype == 'charcnn':
model = char_cnn_model(params)
elif params.nntype == 'charagram':
model = charagram_model(params)
else:
"Error no type specified"
utils.train(model, examples, params)
print('updating net...')
net.collect_params().reset_ctx(ctx)
em = array(em, ctx=mx.cpu())
net.collect_params()['sequential0_embedding0_weight'].set_data(em)
net.collect_params()['sequential0_embedding0_weight'].grad_req = 'null'
print_batches = 100
shuffle_idx = np.random.permutation(train_data.shape[0])
train_data = train_data[shuffle_idx]
train_label = train_label[shuffle_idx]
data_iter = NDArrayIter(data=train_data[:-5000],
label=train_label[:-5000],
batch_size=args.batch_size,
shuffle=True)
val_data_iter = NDArrayIter(data=train_data[-5000:],
label=train_label[-5000:],
batch_size=args.batch_size,
shuffle=False)
trainer = Trainer(net.collect_params(), 'adam', {'learning_rate': 0.001})
# trainer = Trainer(net.collect_params(),'RMSProp', {'learning_rate': 0.001})
print("training data")
utils.train(data_iter,
val_data_iter,
net,
EntropyLoss,
trainer,
ctx,
num_epochs=args.epochs,
print_batches=print_batches)
)
args = parser.parse_args()
params.LW = args.LW
params.outfile = args.outfile
params.batchsize = args.batchsize
params.dim = args.dim
params.wordfile = args.wordfile
params.save = str2bool(args.save)
params.train = args.train
params.margin = args.margin
params.type = args.samplingtype
params.epochs = args.epochs
params.evaluate = str2bool(args.evaluate)
params.learner = str2learner(args.learner)
params.learner = lasagne.updates.adagrad
(words, We) = getWordmap(params.wordfile)
examples = getData(params.train, words)
if args.num_examples:
examples = examples[0 : args.num_examples]
print "Number of training examples: ", len(examples)
print sys.argv
model = paragram_word_model(We, params)
train(model, examples, words, params)
# Quit here if dry run
if args.dry:
exit(0)
# -------------------------------------------------------------------------
# Training
# -------------------------------------------------------------------------
if args.training:
opt = torch.optim.Adam(model.parameters(), lr=0.01)
loss = F.binary_cross_entropy_with_logits
train(model,
shuf_dataloader,
opt,
loss,
args)
# -------------------------------------------------------------------------
# Inference
# -------------------------------------------------------------------------
if args.inference:
loss = F.binary_cross_entropy_with_logits
preds = []
def collector(outputs, targets):
preds.append(outputs.detach().cpu().numpy())
loss_q = ((self.q(states, actions) - y) ** 2).mean()
loss_pi = -(self.q(states, self.pi(states))).mean()
# Update pi
self.optim_pi.zero_grad()
loss_pi.backward()
self.optim_pi.step()
# Update q
self.optim_q.zero_grad()
loss_q.backward()
self.optim_q.step()
# Copy to the target networks
with torch.no_grad():
# Copy to the targets networks
for p_target, p in zip(self.pi_target.parameters(), self.pi.parameters()):
p_target.mul_(polyak_factor)
p_target.add_((1 - polyak_factor) * p)
for p_target, p in zip(self.q_target.parameters(), self.q.parameters()):
p_target.mul_(polyak_factor)
p_target.add_((1 - polyak_factor) * p)
def evaluate(self):
self.noise_scale = 0
train(gym.make('Pendulum-v0'), Agent, plot_frequency=10, tests=5)
def train_private_models(args):
"""Train N = num-models private models."""
assert 0 <= args.begin_id and args.begin_id < args.end_id and args.end_id <= args.num_models
# Logs
filename = 'logs-(id:{:d}-{:d})-(num-epochs:{:d}).txt'.format(
args.begin_id + 1, args.end_id, args.num_epochs)
file = open(os.path.join(args.private_model_path, filename), 'w')
utils.augmented_print("##########################################", file)
utils.augmented_print(
"Training private models on '{}' dataset!".format(args.dataset), file)
utils.augmented_print(
"Training private models on '{}' architecture!".format(
args.architecture), file)
utils.augmented_print(
"Number of private models: {:d}".format(args.num_models), file)
utils.augmented_print("Initial learning rate: {:.2f}".format(args.lr), file)
utils.augmented_print(
"Number of epochs for training each model: {:d}".format(
args.num_epochs), file)
# Dataloaders
if args.dataset_type == 'imbalanced':
all_private_trainloaders = utils.load_private_data_imbalanced(args)
elif args.dataset_type == 'balanced':
all_private_trainloaders = utils.load_private_data(args)
else:
raise Exception('Unknown dataset type: {}'.format(args.dataset_type))
evalloader = utils.load_evaluation_data(args)
# Training
all_acc = []
for id in range(args.begin_id, args.end_id):
utils.augmented_print("##########################################",
file)
# Private model
model = models.Private_Model('model({:d})'.format(id + 1), args)
if args.cuda:
model.cuda()
trainloader = all_private_trainloaders[id]
counts, ratios = utils.class_ratio(trainloader.dataset, args)
utils.augmented_print(
"Label counts: {}".format(np.array2string(counts, separator=', ')),
file)
utils.augmented_print("Class ratios: {}".format(
np.array2string(ratios, precision=2, separator=', ')), file)
utils.augmented_print(
"Number of samples: {:d}".format(len(trainloader.dataset)), file)
utils.augmented_print("Steps per epoch: {:d}".format(len(trainloader)),
file)
# Optimizer
optimizer = optim.SGD(model.parameters(), lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
scheduler = get_scheduler(args=args, optimizer=optimizer)
# Training steps
for epoch in range(args.num_epochs):
train_loss = utils.train(model, trainloader, optimizer, args)
# Scheduler step is based only on the train data, we do not use the
# test data to schedule the decrease in the learning rate.
scheduler.step(train_loss)
eval_loss, acc = utils.evaluate(model, evalloader, args)
acc_detailed = utils.evaluate_detailed(model, evalloader, args)
all_acc.append(acc)
utils.augmented_print(
"Model '{}' | Evaluation Loss: {:.4f} | Accuracy: {:.2f}% | Detailed Accuracy(%): {}".format(
model.name, eval_loss, acc,
np.array2string(acc_detailed, precision=2, separator=', ')),
file, flush=True)
# Checkpoints
state = {'epoch': args.num_epochs, 'accuracy': acc,
'eval_loss': eval_loss, 'state_dict': model.state_dict()}
filename = "checkpoint-{}.pth.tar".format(model.name)
filepath = os.path.join(args.private_model_path, filename)
torch.save(state, filepath)
utils.augmented_print("##########################################", file)
utils.augmented_print(
"Average accuracy of private models: {:.2f}%".format(np.mean(all_acc)),
file)
utils.augmented_print("##########################################", file)
file.close()
def run_model(model_name, data_dict, cuda):
print("running ", model_name)
if model_name not in IMPLEMENTED_MODELS:
NotImplementedError(
"You must choose one of these:{}".format(IMPLEMENTED_MODELS))
else:
emb_matrix = data_dict['emb_matrix']
train_batches = data_dict['train_batches']
val_batches = data_dict['val_batches']
test_batches = data_dict['test_batches']
set_seeds(SEED)
harassment_f1_scores = []
harassment_recall_scores = []
harassment_precision_scores = []
indirect_f1_scores = []
indirect_recall_scores = []
indirect_precision_scores = []
sexual_f1_scores = []
sexual_recall_scores = []
sexual_precision_scores = []
physical_f1_scores = []
physical_recall_scores = []
physical_precision_scores = []
runs = CONFIG['iterations']
for i in range(1, runs + 1):
print("***** iteration: ", i)
if model_name == "vanilla_projected_last":
model = ProjectedVanillaRNN(
emb_matrix, embeddings_dropout=CONFIG['dropout'])
elif model_name == "vanilla_projected_avg":
model = ProjectedVanillaRNN(
emb_matrix,
avg_pooling=True,
embeddings_dropout=CONFIG['dropout'])
elif model_name == "multi_attention":
model = MultiAttentionRNN(
emb_matrix,
embeddings_dropout=CONFIG['dropout'],
trainable_embeddings=CONFIG['trainable_embeddings'])
elif model_name == "multi_projected_attention":
model = ProjectedMultiAttentionRNN(
emb_matrix, embeddings_dropout=CONFIG['dropout'])
elif model_name == "attention":
model = AttentionRNN(
emb_matrix,
embeddings_dropout=CONFIG['dropout'],
trainable_embeddings=CONFIG['trainable_embeddings'])
elif model_name == "projected_attention":
model = ProjectedAttentionRNN(
emb_matrix, embeddings_dropout=CONFIG['dropout'])
elif model_name == "vanilla_avg":
model = VanillaRnn(
emb_matrix,
avg_pooling=True,
embeddings_dropout=CONFIG['dropout'],
trainable_embeddings=CONFIG['trainable_embeddings'])
else:
model = VanillaRnn(
emb_matrix,
embeddings_dropout=CONFIG['dropout'],
trainable_embeddings=CONFIG['trainable_embeddings'])
optimizer = Adam(model.params, CONFIG['lr'])
criterion = BCEWithLogitsLoss()
train(model=model,
train_batches=train_batches,
test_batches=val_batches,
optimizer=optimizer,
criterion=criterion,
epochs=CONFIG['epochs'],
init_patience=CONFIG['patience'],
cuda=cuda)
model = load_model(model)
d = generate_results(model, test_batches, cuda)
df = generate_test_submission_values(
harassment_dict=d['harassment'],
sexual_dict=d['sexual'],
physical_dict=d['physical'],
indirect_dict=d['indirect'])
df_results = pd.read_csv(TEST_RESULTS)
harassment_f1_scores.append(
f1_score(df_results.harassment.values, df.Harassment.values))
harassment_precision_scores.append(
precision_score(df_results.harassment.values,
df.Harassment.values))
harassment_recall_scores.append(
recall_score(df_results.harassment.values,
df.Harassment.values))
indirect_f1_scores.append(
f1_score(df_results.IndirectH.values, df.IndirectH.values))
indirect_precision_scores.append(
precision_score(df_results.IndirectH.values,
df.IndirectH.values))
indirect_recall_scores.append(
recall_score(df_results.IndirectH.values, df.IndirectH.values))
sexual_f1_scores.append(
f1_score(df_results.SexualH.values, df.SexualH.values))
sexual_precision_scores.append(
precision_score(df_results.SexualH.values, df.SexualH.values))
sexual_recall_scores.append(
recall_score(df_results.SexualH.values, df.SexualH.values))
physical_f1_scores.append(
f1_score(df_results.PhysicalH.values, df.PhysicalH.values))
physical_precision_scores.append(
precision_score(df_results.PhysicalH.values,
df.PhysicalH.values))
physical_recall_scores.append(
recall_score(df_results.PhysicalH.values, df.PhysicalH.values))
results_dict = {
'model': [model_name for _ in range(runs)],
'harassment_f1_score': harassment_f1_scores,
'harassment_recall': harassment_recall_scores,
'harassment_precision': harassment_precision_scores,
'indirect_f1_score': indirect_f1_scores,
'indirect_recall': indirect_recall_scores,
'indirect_precision': indirect_precision_scores,
'sexual_f1_score': sexual_f1_scores,
'sexual_recall': sexual_recall_scores,
'sexual_precision': sexual_precision_scores,
'physical_f1_score': physical_f1_scores,
'physical_recall': physical_recall_scores,
'physical_precision': physical_precision_scores
}
df = pd.DataFrame.from_dict(results_dict)
if "results.csv" in os.listdir(RESULTS_DIR):
df_old = pd.read_csv(RESULTS_DIR + "results.csv")
df = pd.concat([df_old, df])
df.to_csv(RESULTS_DIR + "results.csv", index=False)
def retrain_private_models(args):
"""Retrain N = num-querying-parties private models."""
assert 0 <= args.begin_id and args.begin_id < args.end_id and args.end_id <= args.num_querying_parties
# Logs
filename = 'logs-(num_models:{:d})-(id:{:d}-{:d})-(num-epochs:{:d})-(budget:{:f})-(dataset:{})-(architecture:{}).txt'.format(
args.num_models,
args.begin_id + 1, args.end_id,
args.num_epochs,
args.budget,
args.dataset,
args.architecture,
)
print('filename: ', filename)
file = open(os.path.join(args.retrained_private_model_path, filename), 'w')
utils.augmented_print("##########################################", file)
utils.augmented_print(
"Retraining the private models of all querying parties on '{}' dataset!".format(
args.dataset), file)
utils.augmented_print(
"Number of querying parties: {:d}".format(args.num_querying_parties),
file)
utils.augmented_print("Initial learning rate: {:.2f}".format(args.lr), file)
utils.augmented_print(
"Number of epochs for retraining each model: {:d}".format(
args.num_epochs), file)
# Dataloaders
# all_augmented_dataloaders = utils.load_private_data_and_qap_imbalanced(args)
all_augmented_dataloaders = utils.load_private_data_and_qap(args)
evalloader = utils.load_evaluation_data(args)
# Training
for id in range(args.begin_id, args.end_id):
utils.augmented_print("##########################################",
file)
# Different random seed
seed_list = [11, 13, 17, 113, 117]
eval_loss_list = []
acc_list = []
acc_detailed_list = []
for seed in seed_list:
args.seed = seed
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if args.cuda:
torch.cuda.manual_seed(args.seed)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
# Private model
model = models.Private_Model('model({:d})'.format(id + 1), args)
if args.cuda:
model.cuda()
trainloader = all_augmented_dataloaders[id]
# Optimizer
optimizer = optim.SGD(model.parameters(), lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
decay_steps = [int(args.num_epochs * 0.5),
int(args.num_epochs * 0.75),
int(args.num_epochs * 0.9)]
# Training steps
for epoch in range(args.num_epochs):
if epoch in decay_steps:
for param_group in optimizer.param_groups:
param_group['lr'] *= 0.1
utils.train(model, trainloader, optimizer, args)
eval_loss, acc = utils.evaluate(model, evalloader, args)
acc_detailed = utils.evaluate_detailed(model, evalloader, args)
eval_loss_list.append(eval_loss)
acc_list.append(acc)
acc_detailed_list.append(acc_detailed)
# # Checkpoints
# state = {'epoch': args.num_epochs, 'accuracy': acc, 'eval_loss': eval_loss, 'state_dict': model.state_dict()}
# filename = "checkpoint-{}.pth.tar".format(model.name)
# filepath = os.path.join(args.retrained_private_model_path, filename)
# torch.save(state, filepath)
eval_loss_list = np.array(eval_loss_list)
acc_list = np.array(acc_list)
acc_detailed_list = np.array(acc_detailed_list)
counts, ratios = utils.class_ratio(trainloader.dataset, args)
utils.augmented_print(
"Label counts: {}".format(np.array2string(counts, separator=', ')),
file)
utils.augmented_print("Class ratios: {}".format(
np.array2string(ratios, precision=2, separator=', ')), file)
utils.augmented_print(
"Number of samples: {:d}".format(len(trainloader.dataset)), file)
utils.augmented_print("Steps per epoch: {:d}".format(len(trainloader)),
file)
utils.augmented_print(
"Model: '{}' | Architecture: '{}' | Dataset: '{}' | Number of models: {:d} | Budget: {:.4f} | Evaluation Loss: {:.4f} | Accuracy: {:.2f}% | Detailed Accuracy(%): {} | Standard Deviation(%): {}".format(
model.name,
args.architecture,
args.dataset,
args.num_models,
args.budget,
eval_loss_list.mean(axis=0), acc_list.mean(axis=0),
np.array2string(acc_detailed_list.mean(axis=0), precision=2,
separator=', '),
np.array2string(acc_detailed_list.std(axis=0), precision=2,
separator=', ')), file, flush=True)
utils.augmented_print("##########################################", file)
file.close()
def main(arg):
sys.stdout = Logger(osp.join(args.logs_dir, 'log_1109_rcan.txt'))
print("====>> Read file list")
file_name = sorted(os.listdir(arg.VIDEO4K_LR))
lr_list = []
hr_list = []
for fi in file_name:
lr_tmp = sorted(glob.glob(arg.VIDEO4K_LR + '/' + fi + '/*.png'))
lr_list.extend(lr_tmp)
hr_tmp = sorted(glob.glob(arg.VIDEO4K_HR + '/' + fi + '/*.png'))
if len(hr_tmp) != 100:
print(fi)
hr_list.extend(hr_tmp)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
torch.cuda.manual_seed(args.seed)
print(len(lr_list))
print(len(hr_list))
cudnn.benchmark = True
print("===> Loading datasets")
data_set = DatasetLoader(lr_list, hr_list, arg.patch_size, arg.scale)
train_loader = DataLoader(data_set,
batch_size=arg.batch_size,
num_workers=arg.workers,
shuffle=True,
pin_memory=True)
print("===> Building model")
device_ids = [0]
model = RCAN(arg)
criterion = nn.L1Loss(size_average=False)
print("===> Setting GPU")
model = nn.DataParallel(model, device_ids=device_ids)
model = model.cuda()
criterion = criterion.cuda()
# optionally resume from a checkpoint
if arg.resume:
if os.path.isfile(args.resume):
print("=> loading checkpoint '{}'".format(arg.resume))
checkpoint = torch.load(arg.resume)
new_state_dict = OrderedDict()
for k, v in checkpoint.items():
namekey = 'module.' + k # remove `module.`
new_state_dict[namekey] = v
# load params
model.load_state_dict(new_state_dict)
else:
print("=> no checkpoint found at '{}'".format(args.resume))
print("===> Setting Optimizer")
optimizer = optim.Adam(filter(lambda p: p.requires_grad,
model.parameters()),
lr=arg.lr,
weight_decay=arg.weight_decay,
betas=(0.9, 0.999),
eps=1e-08)
print("===> Training")
for epoch in range(args.start_epoch, args.epochs):
adjust_lr(optimizer, epoch)
train(
train_loader,
optimizer,
model,
criterion,
epoch,
arg,
len(hr_list),
)
save_checkpoint(model, epoch)
from torch.optim import Adam
from audio import wav_to_spectrogram
from datasets import LJSpeechDataset
from models import MelSpectrogramNet
from text import text_to_sequence
from utils import train
from torch.utils.data.dataset import Subset
num_epochs = 10
batch_size = 1
PATH = '/home/austin/data/tacotron/LJSpeech-1.0'
dataset = LJSpeechDataset(path=PATH, text_transforms=text_to_sequence,
audio_transforms=wav_to_spectrogram)
dataset = Subset(dataset, range(1))
melnet = MelSpectrogramNet()
melnet.cuda()
optimizer = Adam(melnet.parameters())
train(melnet, optimizer, dataset, num_epochs, batch_size)
search_dir = conjugate_gradient(HVP, g)
max_length = torch.sqrt(2 * delta / (search_dir @ HVP(search_dir)))
max_step = max_length * search_dir
def criterion(step):
apply_update(step)
with torch.no_grad():
distribution_new = actor(states)
distribution_new = torch.distributions.utils.clamp_probs(distribution_new)
probabilities_new = distribution_new[range(distribution_new.shape[0]), actions]
L_new = surrogate_loss(probabilities_new, probabilities, advantages)
KL_new = kl_div(distribution, distribution_new)
L_improvement = L_new - L
if L_improvement > 0 and KL_new <= delta:
return True
apply_update(-step)
return False
line_search(max_step, criterion, max_iterations=10)
# Train using our get_action() and update() functions
train(env, get_action, update_agent, num_rollouts=10, render_frequency=None, print_frequency=10,
plot_frequency=None, epochs=1000)
patience=args['patience'],
verbose=True)
module[idx].load_state_dict(checkpoint['model_state_dict'])
if args['load_optimizer']:
optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
if args['load_scheduler']:
scheduler.load_state_dict(checkpoint['scheduler_state_dict'])
if len(module) == 1:
module = module[0]
else:
module = model.Ensemble(module, args)
if args['mode'] == 'train':
logger.info('Starting training.')
utils.train(module, optimizer, scheduler, train_iter, val_iter, args)
elif args['mode'] == 'eval':
logger.info('Starting evaluation.')
evaluation_results = {}
# evaluation_results['train'] = utils.eval(module, train_iter, args)
evaluation_results['valid'] = utils.eval(module,
val_iter,
args,
write_to_file=True)
logger.info('\n' + pprint.pformat(evaluation_results), args)
elif args['mode'] == 'test':
logger.info('Starting testing.')
utils.predict_write_to_file(module, test_iter, args)
logger.info('Done writing predictions to file.')
# Import the necessary packages
import pandas as pd
import argparse
import cv2
from utils import classify, describe, train
# Load Random Forest Classifier
model = train()
# Set up the argument parser
ap = argparse.ArgumentParser()
# Define arguments
ap.add_argument("-f", "--file", required=True, help="Path to the pandas dataframe with Instagram metadata.")
# Parse arguments
args = vars(ap.parse_args())
# Assign arguments to variables
df_file = args["file"]
# Load dataframe
df = pd.read_pickle(df_file)
print "*** Loading Instagram metadata from {} ...".format(df_file)
# Loop over images in the dataframe
for index, row in df.iterrows():
# Define path
ipath = "test_output/" + row['Filename']
def main(train_file,
valid_file,
test_file,
embeddings_file,
target_dir,
hidden_size=300,
dropout=0.5,
num_classes=2,
epochs=64,
batch_size=32,
lr=0.0004,
patience=5,
max_grad_norm=10.0,
checkpoint=None,
proportion=1,
output=None):
"""
Train the ESIM model on the SNLI dataset.
Args:
train_file: A path to some preprocessed data that must be used
to train the model.
valid_file: A path to some preprocessed data that must be used
to validate the model.
embeddings_file: A path to some preprocessed word embeddings that
must be used to initialise the model.
target_dir: The path to a directory where the trained model must
be saved.
hidden_size: The size of the hidden layers in the model. Defaults
to 300.
dropout: The dropout rate to use in the model. Defaults to 0.5.
num_classes: The number of classes in the output of the model.
Defaults to 3.
epochs: The maximum number of epochs for training. Defaults to 64.
batch_size: The size of the batches for training. Defaults to 32.
lr: The learning rate for the optimizer. Defaults to 0.0004.
patience: The patience to use for early stopping. Defaults to 5.
checkpoint: A checkpoint from which to continue training. If None,
training starts from scratch. Defaults to None.
"""
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
print(20 * "=", " Preparing for training ", 20 * "=")
if not os.path.exists(target_dir):
os.makedirs(target_dir)
# -------------------- Data loading ------------------- #
print("\t* Loading training data...")
with open(train_file, "rb") as pkl:
train_data = NLIDataset(
pickle.load(pkl), proportion, isRandom=True
) #training data will be shuffled first, then we will get random data of different proportion
train_loader = DataLoader(train_data, shuffle=False, batch_size=batch_size)
print("\t* Loading validation data...")
with open(valid_file, "rb") as pkl:
valid_data = NLIDataset(pickle.load(pkl))
valid_loader = DataLoader(valid_data, shuffle=False, batch_size=batch_size)
print("\t* Loading test data...")
with open(test_file, "rb") as pkl:
test_data = NLIDataset(pickle.load(pkl))
test_loader = DataLoader(test_data, shuffle=False, batch_size=batch_size)
# -------------------- Model definition ------------------- #
print("\t* Building model...")
with open(embeddings_file, "rb") as pkl:
embeddings = torch.tensor(pickle.load(pkl), dtype=torch.float)\
.to(device)
model = ESIM(embeddings.shape[0],
embeddings.shape[1],
hidden_size,
embeddings=embeddings,
dropout=dropout,
num_classes=num_classes,
device=device).to(device)
# -------------------- Preparation for training ------------------- #
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=lr)
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer,
mode="max",
factor=0.5,
patience=0)
best_score = 0.0
start_epoch = 1
# Data for loss curves plot.
epochs_count = []
train_losses = []
valid_losses = []
# Continuing training from a checkpoint if one was given as argument.
if checkpoint:
checkpoint = torch.load(checkpoint)
start_epoch = checkpoint["epoch"] + 1
best_score = checkpoint["best_score"]
print("\t* Training will continue on existing model from epoch {}...".
format(start_epoch))
model.load_state_dict(checkpoint["model"])
optimizer.load_state_dict(checkpoint["optimizer"])
epochs_count = checkpoint["epochs_count"]
train_losses = checkpoint["train_losses"]
valid_losses = checkpoint["valid_losses"]
# Compute loss and accuracy before starting (or resuming) training.
_, valid_loss, valid_accuracy = validate(model, valid_loader, criterion)
print("\t* Validation loss before training: {:.4f}, accuracy: {:.4f}%".
format(valid_loss, (valid_accuracy * 100)))
# -------------------- Training epochs ------------------- #
print("\n", 20 * "=", "Training ESIM model on device: {}".format(device),
20 * "=")
patience_counter = 0
for epoch in range(start_epoch, epochs + 1):
epochs_count.append(epoch)
print("* Training epoch {}:".format(epoch))
epoch_time, epoch_loss, epoch_accuracy = train(model, train_loader,
optimizer, criterion,
epoch, max_grad_norm)
train_losses.append(epoch_loss)
print("-> Training time: {:.4f}s, loss = {:.4f}, accuracy: {:.4f}%".
format(epoch_time, epoch_loss, (epoch_accuracy * 100)))
print("* Validation for epoch {}:".format(epoch))
epoch_time, epoch_loss, epoch_accuracy = validate(
model, valid_loader, criterion)
valid_losses.append(epoch_loss)
print("-> Valid. time: {:.4f}s, loss: {:.4f}, accuracy: {:.4f}%\n".
format(epoch_time, epoch_loss, (epoch_accuracy * 100)))
# Update the optimizer's learning rate with the scheduler.
scheduler.step(epoch_accuracy)
print("* Testing for epoch {}:".format(epoch))
batch_time, total_time, f1, accuracy = test(model, num_classes,
test_loader)
print(
"-> Average batch processing time: {:.4f}s, total test time: {:.4f}s, f1: {:.4f}, accuracy: {:.4f}%"
.format(batch_time, total_time, f1, (accuracy * 100)))
print(20 * "====")
# Early stopping on validation accuracy.
if epoch > 2:
if epoch_accuracy <= best_score:
patience_counter += 1
else:
best_score = epoch_accuracy
patience_counter = 0
# Save the best model. The optimizer is not saved to avoid having
# a checkpoint file that is too heavy to be shared. To resume
# training from the best model, use the 'esim_*.pth.tar'
# checkpoints instead.
torch.save(
{
"epoch": epoch,
"model": model.state_dict(),
"best_score": best_score,
"epochs_count": epochs_count,
"train_losses": train_losses,
"valid_losses": valid_losses
},
os.path.join(
target_dir,
output + "_" + str(proportion) + "_best.pth.tar"))
if patience_counter >= patience:
print("-> Early stopping: patience limit reached, stopping...")
checkpoint = torch.load(
os.path.join(
target_dir,
output + "_" + str(proportion) + "_best.pth.tar"))
# Retrieving model parameters from checkpoint.
vocab_size = checkpoint["model"][
"_word_embedding.weight"].size(0)
embedding_dim = checkpoint["model"][
'_word_embedding.weight'].size(1)
hidden_size = checkpoint["model"]["_projection.0.weight"].size(
0)
num_classes = checkpoint["model"][
"_classification.4.weight"].size(0)
print("\t* Final test...")
model = ESIM(vocab_size,
embedding_dim,
hidden_size,
num_classes=num_classes,
device=device).to(device)
model.load_state_dict(checkpoint["model"])
batch_time, total_time, f1, accuracy = test(
model, num_classes, test_loader, print_Confusion=True)
print("-> Final f1, accuracy: {:.4f}, {:.4f}%".format(
f1, accuracy * 100))
os.remove(
os.path.join(
target_dir,
output + "_" + str(proportion) + "_best.pth.tar"))
break
if epoch == 15:
checkpoint = torch.load(
os.path.join(target_dir,
output + "_" + str(proportion) + "_best.pth.tar"))
# Retrieving model parameters from checkpoint.
vocab_size = checkpoint["model"]["_word_embedding.weight"].size(0)
embedding_dim = checkpoint["model"]['_word_embedding.weight'].size(
1)
hidden_size = checkpoint["model"]["_projection.0.weight"].size(0)
num_classes = checkpoint["model"]["_classification.4.weight"].size(
0)
print("\t* Final test...")
model = ESIM(vocab_size,
embedding_dim,
hidden_size,
num_classes=num_classes,
device=device).to(device)
model.load_state_dict(checkpoint["model"])
batch_time, total_time, f1, accuracy = test(model,
num_classes,
test_loader,
print_Confusion=True)
print("-> Final f1, accuracy: {:.4f}, {:.4f}%".format(
f1, accuracy * 100))
os.remove(
os.path.join(target_dir,
output + "_" + str(proportion) + "_best.pth.tar"))
import telebot
import random
from utils import train, generate_sentence
import phrases
import re
import config
bot = telebot.TeleBot(config.TOKEN, threaded=False)
model = train('corpus.txt')
@bot.message_handler(commands=['start'])
def start_command(message):
bot.send_message(message.chat.id, phrases.start)
@bot.message_handler(commands=['help'])
def help_command(message):
bot.send_message(message.chat.id, phrases.helpa)
regEng = re.compile('[a-zA-Z]+')
@bot.message_handler(content_types=['text'])
def reply(message):
if regEng.search(message.text) is not None:
reply = random.choice(phrases.english)
else:
s = message.text.lower()
s = s.split()
s = s[-1].rstrip('.,:;?!')
try:
reply = generate_sentence(model,s)
else:
device_ids = None
model = nn.DataParallel(model, device_ids=device_ids).cuda()
criterion = CrossEntropyLoss()
train_kwargs = dict(
args=args,
model=model,
criterion=criterion,
train_loader=train_loader,
valid_loader=valid_loader,
validation=validation,
patience=10,
)
if getattr(model, 'finetune', None):
utils.train(
init_optimizer=lambda lr: SGD(model.net.fc.parameters(), lr=lr, momentum=0.9),
n_epochs=1,
**train_kwargs)
utils.train(
init_optimizer=lambda lr: SGD(model.parameters(), lr=lr, momentum=0.9),
**train_kwargs)
else:
utils.train(
init_optimizer=lambda lr: SGD(model.parameters(), lr=lr, momentum=0.9),
**train_kwargs)
# train loglasso model per week import utils utils.train()
model_name = 'model.pth'
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
print(f"Found {device} ...")
print("Instantiating RNN Model")
if not os.path.exists(log_dir):
os.mkdir(log_dir)
model_save_path = os.path.join(log_dir, model_name)
model = RNNModel(x_train.shape[-1], hidden_dim, num_layers,
y_train.shape[-1]).to(device)
optimizer = optim.Adam(model.parameters(), lr=lr)
criterion = nn.MSELoss()
print("< Training starts >")
model = train(model, dataloader_train, dataloader_val, device, criterion,
optimizer, n_epochs, model_save_path)
print("Testing on test data-set ")
log_dir = './ckpt'
model_name = 'model.pth'
model_save_path = os.path.join(log_dir, model_name)
output_dim = 4
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
model = RNNModel(x_test.shape[-1], hidden_dim, num_layers,
output_dim).to(device)
y_test_pred = test(x_test, model, model_save_path, device)
ft_dict = {
0: 'open',
1: 'high',
2: 'low',
3: 'close',
criterion = MultiLabelSoftMarginLoss()
train_kwargs = dict(
args=args,
model=model,
criterion=criterion,
train_loader=train_loader,
valid_loader=valid_loader,
validation=validation,
patience=4,
)
if getattr(model, 'finetune', None):
utils.train(
init_optimizer=lambda lr: Adam(model.net.fc.parameters(), lr=lr),
n_epochs=1,
**train_kwargs)
utils.train(
init_optimizer=lambda lr: Adam(model.parameters(), lr=lr),
**train_kwargs)
else:
utils.train(
init_optimizer=lambda lr: Adam(model.parameters(), lr=lr),
**train_kwargs)
#
#
# utils.train(
# init_optimizer=lambda lr: Adam(model.parameters(), lr=lr),
def model_train_validate_test(train_df,
dev_df,
test_df,
target_dir,
max_seq_len=64,
num_labels=2,
epochs=10,
batch_size=32,
lr=2e-05,
patience=1,
max_grad_norm=10.0,
if_save_model=True,
checkpoint=None):
bertmodel = RobertModel(requires_grad=True, num_labels=num_labels)
tokenizer = bertmodel.tokenizer
print(20 * "=", " Preparing for training ", 20 * "=")
# 保存模型的路径，没有则创建文件夹
if not os.path.exists(target_dir):
os.makedirs(target_dir)
# -------------------- Data loading ------------------- #
print("\t* Loading training data...")
train_data = DataPrecessForSentence(tokenizer, train_df, max_seq_len)
train_loader = DataLoader(train_data, shuffle=True, batch_size=batch_size)
print("\t* Loading validation data...")
dev_data = DataPrecessForSentence(tokenizer, dev_df, max_seq_len)
dev_loader = DataLoader(dev_data, shuffle=True, batch_size=batch_size)
print("\t* Loading test data...")
test_data = DataPrecessForSentence(tokenizer, test_df, max_seq_len)
test_loader = DataLoader(test_data, shuffle=False, batch_size=batch_size)
# -------------------- Model definition ------------------- #
print("\t* Building model...")
device = torch.device("cuda")
model = bertmodel.to(device)
total_params = sum(p.numel() for p in model.parameters())
print(f'{total_params:,} total parameters.')
total_trainable_params = sum(p.numel() for p in model.parameters()
if p.requires_grad)
print(f'{total_trainable_params:,} training parameters.')
# -------------------- Preparation for training ------------------- #
# 待优化的参数
param_optimizer = list(model.named_parameters())
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [{
'params':
[p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],
'weight_decay':
0.01
}, {
'params':
[p for n, p in param_optimizer if any(nd in n for nd in no_decay)],
'weight_decay':
0.0
}]
optimizer = AdamW(optimizer_grouped_parameters, lr=lr)
# 当网络的评价指标不在提升的时候，可以通过降低网络的学习率来提高网络性能
warmup_steps = math.ceil(len(train_loader) * epochs * 0.1)
total_steps = len(train_loader) * epochs
scheduler = get_linear_schedule_with_warmup(optimizer,
num_warmup_steps=0,
num_training_steps=total_steps)
# scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode="max", factor=0.85, patience=0)
best_score = 0.0
start_epoch = 1
# Data for loss curves plot
epochs_count = []
train_losses = []
train_accuracies = []
valid_losses = []
valid_accuracies = []
# Continuing training from a checkpoint if one was given as argument
if checkpoint:
checkpoint = torch.load(checkpoint)
start_epoch = checkpoint["epoch"] + 1
best_score = checkpoint["best_score"]
print("\t* Training will continue on existing model from epoch {}...".
format(start_epoch))
model.load_state_dict(checkpoint["model"])
optimizer.load_state_dict(checkpoint["optimizer"])
epochs_count = checkpoint["epochs_count"]
train_losses = checkpoint["train_losses"]
train_accuracy = checkpoint["train_accuracy"]
valid_losses = checkpoint["valid_losses"]
valid_accuracy = checkpoint["valid_accuracy"]
# Compute loss and accuracy before starting (or resuming) training.
_, valid_loss, valid_accuracy, _, = validate(model, dev_loader)
print("\n* Validation loss before training: {:.4f}, accuracy: {:.4f}%".
format(valid_loss, (valid_accuracy * 100)))
# -------------------- Training epochs ------------------- #
print("\n", 20 * "=",
"Training roberta model on device: {}".format(device), 20 * "=")
patience_counter = 0
for epoch in range(start_epoch, epochs + 1):
epochs_count.append(epoch)
print("* Training epoch {}:".format(epoch))
epoch_time, epoch_loss, epoch_accuracy = train(model, train_loader,
optimizer, epoch,
max_grad_norm)
train_losses.append(epoch_loss)
train_accuracies.append(epoch_accuracy)
print("-> Training time: {:.4f}s, loss = {:.4f}, accuracy: {:.4f}%".
format(epoch_time, epoch_loss, (epoch_accuracy * 100)))
print("* Validation for epoch {}:".format(epoch))
epoch_time, epoch_loss, epoch_accuracy, _, = validate(
model, dev_loader)
valid_losses.append(epoch_loss)
valid_accuracies.append(epoch_accuracy)
print("-> Valid. time: {:.4f}s, loss: {:.4f}, accuracy: {:.4f}%\n".
format(epoch_time, epoch_loss, (epoch_accuracy * 100)))
# Update the optimizer's learning rate with the scheduler.
# scheduler.step(epoch_accuracy)
scheduler.step()
# Early stopping on validation accuracy.
if epoch_accuracy < best_score:
patience_counter += 1
else:
best_score = epoch_accuracy
patience_counter = 0
if (if_save_model):
torch.save(
{
"epoch": epoch,
"model": model.state_dict(),
"optimizer": optimizer.state_dict(),
"best_score": best_score, # 验证集上的最优准确率
"epochs_count": epochs_count,
"train_losses": train_losses,
"train_accuracy": train_accuracies,
"valid_losses": valid_losses,
"valid_accuracy": valid_accuracies
},
os.path.join(target_dir, "best.pth.tar"))
print("save model succesfully!\n")
print("* Test for epoch {}:".format(epoch))
_, _, test_accuracy, predictions = validate(model, test_loader)
print("Test accuracy: {:.4f}%\n".format(test_accuracy))
test_prediction = pd.DataFrame({'prediction': predictions})
test_prediction.to_csv(os.path.join(target_dir,
"test_prediction.csv"),
index=False)
if patience_counter >= patience:
print("-> Early stopping: patience limit reached, stopping...")
break
def main():
net = get_net()
trainIter, testIter = get_train_test()
trainer,loss = get_trainer(net)
utils.train(trainIter, testIter, net, loss, trainer, utils.try_gpu(), 1000,print_batches=100)
train=False,
download=True,
transform=transform_test)
#testloader = torch.utils.data.DataLoader(testset, batch_size=100, shuffle=False, num_workers=2)
has_bn = utils.check_bn(model)
test_res = {'loss': None, 'accuracy': None, 'nll': None}
for epoch in range(start_epoch, args.epochs + 1):
time_ep = time.time()
lr = learning_rate_schedule(args.lr, epoch, args.epochs)
# lr = args.lr
utils.adjust_learning_rate(optimizer, lr)
train_res = utils.train(loaders['train'], model, optimizer, criterion,
regularizer)
test_res = utils.test(loaders['test'], model, criterion, regularizer)
test_poison_res = utils.test_poison(testset, model, criterion, regularizer)
if epoch % args.save_freq == 0:
utils.save_checkpoint(args.dir,
epoch,
model_state=model.state_dict(),
optimizer_state=optimizer.state_dict())
time_ep = time.time() - time_ep
values = [
epoch, lr, train_res['loss'], train_res['accuracy'], test_res['nll'],
test_res['accuracy'], test_poison_res['nll'],
if hp.embedding == None:
use_pretrained = True
encoder = Encoder(source_vocab_size=len(inp_lang.vocab),
embed_dim=hp.embed_dim,
hidden_dim=hp.hidden_dim,
n_layers=hp.n_layers,
dropout=hp.dropout,
use_pretrained=use_pretrained)
decoder = Decoder(target_vocab_size=len(opt_lang.vocab),
embed_dim=hp.embed_dim,
hidden_dim=hp.hidden_dim,
n_layers=hp.n_layers,
dropout=hp.dropout,
use_pretrained=use_pretrained)
seq2seq = Seq2Seq(encoder, decoder)
seq2seq.to(hp.device)
optimizer = Adam(seq2seq.parameters(), lr=hp.max_lr)
scheduler = SGDRScheduler(optimizer,
max_lr=hp.max_lr,
cycle_length=hp.cycle_length)
train(seq2seq,
optimizer,
scheduler,
train_iter,
val_iter,
num_epochs=hp.num_epochs)
evaluate_metrics(seq2seq, test_iter)
logfile=logfile)
print()
logfile.write('\n')
if (i + 1) % args.save_freq == 0:
saver.save(sess, os.path.join(savedir, 'model'))
logfile.close()
saver.save(sess, os.path.join(savedir, 'model'))
def test():
sess = tf.Session()
saver = tf.train.Saver(tnet['weights'])
saver.restore(sess, os.path.join(savedir, 'model'))
logger = Accumulator('cent', 'acc')
to_run = [tnet['cent'], tnet['acc']]
for j in range(n_test_batches):
bx, by = mnist.test.next_batch(batch_size)
logger.accum(sess.run(to_run, {x: bx, y: by}))
logger.print_(header='test')
if __name__ == '__main__':
if args.mode == 'train':
train()
elif args.mode == 'test':
test()
else:
raise ValueError('Invalid mode %s' % args.mode)
