def prepare_bson(args):
# Dataset
train_bson_path = os.path.join(args.root, 'train_example.bson')
categories_df = pd.read_csv(os.path.join(args.root, 'categories.csv'),
index_col=0)
cat2idx, idx2cat = make_category_tables(categories_df)
offsets_df = pd.read_csv(os.path.join(args.root, 'train_offsets.csv'),
index_col=0)
train_images_df = pd.read_csv(os.path.join(args.root, 'train_images.csv'),
index_col=0)
val_images_df = pd.read_csv(os.path.join(args.root, 'val_images.csv'),
index_col=0)
bson_file = open(train_bson_path, 'rb')
lock = threading.Lock()
train = PreprocessedDataset(bson_file, train_images_df, offsets_df,
cat2idx, lock)
val = PreprocessedDataset(bson_file, val_images_df, offsets_df, cat2idx,
lock)
# Iterator
train_iter = chainer.iterators.SerialIterator(train, args.batchsize)
test_iter = chainer.iterators.SerialIterator(val,
args.batchsize,
repeat=False,
shuffle=False)
return train_iter, test_iter
def main():
parser = argparse.ArgumentParser(
description=
'Extract features from trained convnet model (only 3 channels image)')
parser.add_argument('test', help='Path to test image-label list file')
parser.add_argument('trainedmodel',
help='Trainedmodel for extacting features')
parser.add_argument('--root',
'-R',
default='.',
help='Root directory path of image files')
parser.add_argument('--out',
'-o',
default='result',
help='Output directory')
args = parser.parse_args()
try:
os.makedirs(args.out)
except OSError:
pass
# Initialize the model to train
model = pickle.load(open(args.trainedmodel, 'rb'))
model.to_gpu()
# Load the datasets
test = PreprocessedDataset(args.test, args.root, 224)
# Extract features and write dat file
imagelist = np.genfromtxt(args.test, dtype=np.str)[:, 0]
N = len(imagelist)
results = []
for i in range(len(test)):
print imagelist[i]
sys.stderr.write('{} / {}\r'.format(i, N))
sys.stderr.flush()
image, label = test.get_example(i)
x = image.reshape((1, ) + image.shape)
y = model(inputs={'data': xp.asarray(x)}, outputs=['fc6'], train=False)
outputs = cuda.to_cpu(y[0].data)
results.append(outputs[0])
np.save(os.path.join(args.out, 'features.npy'), results)
def main(args):
assert((args.depth - args.block - 1) % args.block == 0)
n_layer = int((args.depth - args.block - 1) / args.block)
if args.dataset == 'cifar10':
train, test = cifar.get_cifar10()
n_class = 10
elif args.dataset == 'cifar100':
train, test = cifar.get_cifar100()
n_class = 100
elif args.dataset == 'SVHN':
raise NotImplementedError()
images = convert.concat_examples(train)[0]
mean = images.mean(axis=(0, 2, 3))
std = images.std(axis=(0, 2, 3))
train = PreprocessedDataset(train, mean, std, random=args.augment)
test = PreprocessedDataset(test, mean, std)
train_iter = chainer.iterators.SerialIterator(
train, args.batchsize / args.split_size)
test_iter = chainer.iterators.SerialIterator(
test, args.batchsize / args.split_size, repeat=False, shuffle=False)
model = chainer.links.Classifier(DenseNet(
n_layer, args.growth_rate, n_class, args.drop_ratio, 16, args.block))
if args.init_model:
serializers.load_npz(args.init_model, model)
chainer.cuda.get_device(args.gpu).use()
model.to_gpu()
optimizer = chainer.optimizers.NesterovAG(lr=args.lr, momentum=0.9)
optimizer.setup(model)
optimizer.add_hook(chainer.optimizer.WeightDecay(args.weight_decay))
updater = StandardUpdater(
train_iter, optimizer, (args.split_size, 'mean'), device=args.gpu)
trainer = training.Trainer(updater, (300, 'epoch'), out=args.dir)
val_interval = (1, 'epoch')
log_interval = (1, 'epoch')
def lr_shift(): # DenseNet specific!
if updater.epoch == 150 or updater.epoch == 225:
optimizer.lr *= 0.1
return optimizer.lr
trainer.extend(Evaluator(
test_iter, model, device=args.gpu), trigger=val_interval)
trainer.extend(extensions.observe_value(
'lr', lambda _: lr_shift()), trigger=(1, 'epoch'))
trainer.extend(extensions.dump_graph('main/loss'))
trainer.extend(extensions.snapshot_object(
model, 'epoch_{.updater.epoch}.model'), trigger=val_interval)
trainer.extend(extensions.snapshot_object(
optimizer, 'epoch_{.updater.epoch}.state'), trigger=val_interval)
trainer.extend(extensions.LogReport(trigger=log_interval))
start_time = time.time()
trainer.extend(extensions.observe_value(
'time', lambda _: time.time() - start_time), trigger=log_interval)
trainer.extend(extensions.PrintReport([
'time', 'epoch', 'iteration', 'main/loss', 'validation/main/loss',
'main/accuracy', 'validation/main/accuracy', 'lr',
]), trigger=log_interval)
trainer.extend(extensions.observe_value(
'graph', lambda _: create_fig(args.dir)),
trigger=(1, 'epoch'), priority=50)
trainer.extend(extensions.ProgressBar(update_interval=10))
trainer.run()
def main(args):
assert((args.depth - args.block - 1) % args.block == 0)
n_layer = (args.depth - args.block - 1) / args.block
if args.dataset == 'cifar10':
mean = numpy.asarray((125.3,123.0,113.9))#from fb.resnet.torch
std = numpy.asarray((63.0, 62.1, 66.7))# Did the std data computed from 0 padding images?
train, test = dataset.EXget_cifar10(scale=255,mean=mean,std=std)
n_class = 10
elif args.dataset == 'cifar100':
mean = numpy.asarray((129.3,124.1,112.4))#from fb.resnet.torch
std = numpy.asarray((68.2, 65.4, 70.4))
train, test = dataset.EXget_cifar100(scale=255,mean=mean,std=std)
n_class = 100
elif args.dataset == 'SVHN':
raise NotImplementedError()
train = PreprocessedDataset(train, random=True)
test = PreprocessedDataset(test)
train_iter = chainer.iterators.MultiprocessIterator(train, args.batchsize)
test_iter = chainer.iterators.MultiprocessIterator(
test, args.batchsize, repeat=False, shuffle=False)
model = chainer.links.Classifier(DenseNet(n_layer, args.growth_rate, n_class, args.drop_ratio, 16, args.block))
if args.init_model:
serializers.load_npz(args.init_model, model)
import EXoptimizers
optimizer = EXoptimizers.originalNesterovAG(lr=args.lr / len(args.gpus), momentum=0.9)
optimizer.setup(model)
optimizer.add_hook(chainer.optimizer.WeightDecay(args.weight_decay))
devices = {'main': args.gpus[0]}
if len(args.gpus) > 1:
for gid in args.gpus[1:]:
devices['gpu%d' % gid] = gid
updater = training.ParallelUpdater(train_iter, optimizer, devices=devices)
trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=args.dir)
val_interval = (1, 'epoch')
log_interval = (1, 'epoch')
def lr_shift(): # DenseNet specific!
if updater.epoch == 151 or updater.epoch == 226:
optimizer.lr *= 0.1
return optimizer.lr
trainer.extend(Evaluator(
test_iter, model, device=args.gpus[0]), trigger=val_interval)
trainer.extend(extensions.observe_value(
'lr', lambda _: lr_shift()), trigger=(1, 'epoch'))
trainer.extend(extensions.dump_graph('main/loss'))
trainer.extend(extensions.snapshot_object(
model, 'epoch_{.updater.epoch}.model'), trigger=val_interval)
trainer.extend(extensions.snapshot_object(
optimizer, 'epoch_{.updater.epoch}.state'), trigger=val_interval)
trainer.extend(extensions.LogReport(trigger=log_interval))
trainer.extend(extensions.observe_lr(), trigger=log_interval)
start_time = time.time()
trainer.extend(extensions.observe_value(
'time', lambda _: time.time() - start_time), trigger=log_interval)
trainer.extend(extensions.PrintReport([
'time', 'epoch', 'iteration', 'main/loss', 'validation/main/loss',
'main/accuracy', 'validation/main/accuracy', 'lr',
]), trigger=log_interval)
trainer.extend(extensions.observe_value(
'graph', lambda _: create_fig(args.dir)), trigger=(2, 'epoch'))
trainer.extend(extensions.ProgressBar(update_interval=10))
trainer.run()
def main(args):
assert ((args.depth - args.block - 1) % args.block == 0)
n_layer = (args.depth - args.block - 1) / args.block
if args.dataset == 'cifar10':
train, test = cifar.get_cifar10()
n_class = 10
elif args.dataset == 'cifar100':
train, test = cifar.get_cifar100()
n_class = 100
elif args.dataset == 'SVHN':
raise NotImplementedError()
mean = numpy.zeros((3, 32, 32), dtype=numpy.float32)
for image, _ in train:
mean += image / len(train)
train = PreprocessedDataset(train, mean, random=True)
test = PreprocessedDataset(test, mean)
train_iter = chainer.iterators.MultiprocessIterator(train, args.batchsize)
test_iter = chainer.iterators.MultiprocessIterator(test,
args.batchsize,
repeat=False,
shuffle=False)
model = chainer.links.Classifier(
DenseNet(n_layer, args.growth_rate, n_class, args.drop_ratio, 16,
args.block))
if args.init_model:
serializers.load_npz(args.init_model, model)
optimizer = chainer.optimizers.MomentumSGD(lr=args.lr / len(args.gpus),
momentum=0.9)
optimizer.setup(model)
optimizer.add_hook(chainer.optimizer.WeightDecay(args.weight_decay))
devices = {'main': args.gpus[0]}
if len(args.gpus) > 2:
for gid in args.gpus[1:]:
devices['gpu%d' % gid] = gid
updater = training.ParallelUpdater(train_iter, optimizer, devices=devices)
trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=args.dir)
val_interval = (1, 'epoch')
log_interval = (1, 'epoch')
eval_model = model.copy()
eval_model.train = False
trainer.extend(extensions.Evaluator(test_iter,
eval_model,
device=args.gpus[0]),
trigger=val_interval)
trainer.extend(extensions.ExponentialShift('lr', args.lr_decay_ratio),
trigger=(args.lr_decay_freq, 'epoch'))
trainer.extend(extensions.dump_graph('main/loss'))
trainer.extend(extensions.snapshot_object(model,
'epoch_{.updater.epoch}.model'),
trigger=val_interval)
trainer.extend(extensions.snapshot_object(optimizer,
'epoch_{.updater.epoch}.state'),
trigger=val_interval)
trainer.extend(extensions.LogReport(trigger=log_interval))
trainer.extend(extensions.observe_lr(), trigger=log_interval)
start_time = time.time()
trainer.extend(extensions.observe_value(
'time', lambda _: time.time() - start_time),
trigger=log_interval)
trainer.extend(extensions.PrintReport([
'time',
'epoch',
'iteration',
'main/loss',
'validation/main/loss',
'main/accuracy',
'validation/main/accuracy',
'lr',
]),
trigger=log_interval)
trainer.extend(extensions.observe_value('graph',
lambda _: create_fig(args.dir)),
trigger=(2, 'epoch'))
trainer.extend(extensions.ProgressBar(update_interval=10))
trainer.run()
def train_model():
archs = {
'mymodel': mymodel.MyModel,
'nin': nin.NIN,
'alex': alex.Alex,
'lenet': lenet5.Lenet5,
'vgg': vgg16.VGG16,
'googlenet': googlenet.GoogLeNet,
'deepface': deepface.DeepFace
}
parser = argparse.ArgumentParser(
description='Training convnet from dataset (only 3 channels image)')
parser.add_argument('train', help='Path to training image-label list file')
parser.add_argument('test', help='Path to test image-label list file')
parser.add_argument('--arch',
'-a',
choices=archs.keys(),
default='nin',
help='Convnet architecture')
parser.add_argument('--epoch',
'-E',
type=int,
default=10,
help='Number of epochs to train')
parser.add_argument('--batchsize',
'-B',
type=int,
default=32,
help='Training minibatch size')
parser.add_argument('--test_batchsize',
'-b',
type=int,
default=250,
help='Test minibatch size')
parser.add_argument('--gpu',
'-g',
type=int,
default=-1,
help='GPU ID (negative value indicates CPU)')
parser.add_argument('--root',
'-R',
default='.',
help='Root directory path of image files')
parser.add_argument('--mean',
'-m',
default='mean.npy',
help='Mean file (computed by compute_mean.py)')
parser.add_argument('--out',
'-o',
default='result',
help='Output directory')
args = parser.parse_args()
print 'GPU: {}'.format(args.gpu)
print '# Minibatch-size: {}'.format(args.batchsize)
print '# epoch: {}'.format(args.epoch)
print ''
# Initialize model to train
model = archs[args.arch]()
if args.gpu >= 0:
chainer.cuda.get_device_from_id(args.gpu).use()
model.to_gpu()
# Load datasets
mean = np.load(args.mean)
train = PreprocessedDataset(args.train, args.root, mean, model.insize)
test = PreprocessedDataset(args.test, args.root, mean, model.insize)
# Set up iterator
train_iter = chainer.iterators.SerialIterator(train, args.batchsize)
test_iter = chainer.iterators.SerialIterator(test,
args.test_batchsize,
repeat=False,
shuffle=False)
# Set up optimizer
optimizer = chainer.optimizers.AdaDelta()
optimizer.setup(model)
# Set up trainer
updater = training.StandardUpdater(train_iter, optimizer, device=args.gpu)
trainer = training.Trainer(updater, (args.epoch, 'epoch'), args.out)
# Copy chain with shared parameters to flip 'train' flag only in test
eval_model = model.copy()
eval_model.train = False
trainer.extend(extensions.Evaluator(test_iter, eval_model,
device=args.gpu))
trainer.extend(extensions.LogReport())
trainer.extend(
extensions.PlotReport(['main/loss', 'validation/main/loss'],
'epoch',
file_name='loss.png'))
trainer.extend(
extensions.PlotReport(['main/accuracy', 'validation/main/accuracy'],
'epoch',
file_name='accuracy.png'))
trainer.extend(
extensions.PrintReport([
'epoch', 'main/loss', 'validation/main/loss', 'main/accuracy',
'validation/main/accuracy', 'elapsed_time'
]))
trainer.extend(extensions.ProgressBar())
# Run trainer
date = datetime.datetime.today()
start_time = time.clock()
trainer.run()
total_time = datetime.timedelta(seconds=time.clock() - start_time)
# Save trained model
print ''
print 'Training has been finished.'
print 'Total training time: {}.'.format(total_time)
print 'Saving the trained model...',
chainer.serializers.save_npz(
os.path.join(args.out, 'model_final_' + args.arch), model)
print '----> done'
info = open(os.path.join(args.out, 'info'), 'a')
info.write('Date: {}.\n'.format(date.strftime("%Y/%m/%d %H:%M:%S")))
info.write('----> Total training time: {}.'.format(total_time))
pretrained_model_args = torch.load(args.model_path)
TEXT = pretrained_model_args['text']
RAWFIELD = data.RawField()
embed_dim = pretrained_model_args['dim']
PAD_IDX = TEXT.vocab.stoi[TEXT.pad_token]
load_model = TEGRU(pretrained_vec=TEXT.vocab.vectors, ntoken=len(TEXT.vocab), d_model=embed_dim,
nhid=512, pad_token_id=PAD_IDX, hidden_dim=256,
te_nlayers=6)
load_model = load_model.to(device)
load_model.load_state_dict(pretrained_model_args['net'])
if args.test_file:
predicts(load_model, args.test_file, TEXT)
else:
LABEL = data.LabelField(dtype=torch.float)
fields = [
('date', RAWFIELD),
('text', TEXT),
('rise', LABEL),
]
test_dataset = PreprocessedDataset(path='final_dataset/preprocessed/dataset_test_pre.csv', fields=fields)
LABEL.build_vocab(test_dataset)
criterion = nn.BCEWithLogitsLoss()
criterion = criterion.to(device)
load_model.load_state_dict(pretrained_model_args['net'])
valid_iterator = data.Iterator(test_dataset, args.batch_size, train=False, sort=False, shuffle=True, device=device)
valid_loss, valid_acc, _ = evaluate(load_model, valid_iterator, criterion, fields=fields, metric=binary_accuracy)
print(f'\t Val. Loss: {valid_loss:.6f} | Val. Accuracy: {valid_acc}')
TEXT = data.Field(sequential=True, lower=True, batch_first=True)
RAWFIELD = data.RawField()
LABEL = data.LabelField(dtype=torch.float)
fields = [
('date', RAWFIELD),
('text', TEXT),
('rise', LABEL),
]
train_dataset = None
val_dataset = None
test_dataset = None
if load_preprocessed_dataset:
dataset = PreprocessedDataset(
path='final_dataset/preprocessed/dataset_train_pre.csv',
fields=fields)
train_dataset, valid_dataset = dataset.split(split_ratio=0.95,
stratified=True,
strata_field='rise')
test_dataset = PreprocessedDataset(
path='final_dataset/preprocessed/dataset_test_pre.csv',
fields=fields)
else:
dataset = PreprocessedDataset(
path='final_dataset/dataset_train_pre.csv', fields=fields)
train_dataset, valid_dataset = dataset.split(split_ratio=0.95,
stratified=True,
strata_field='rise')
test_dataset = PreprocessedDataset(
path='final_dataset/dataset_test_pre.csv', fields=fields)