def verify(batch_size):
print('verifying batch size: ', batch_size)
fold = Fold()
num_samples = 100
inputs = []
fold_preds = []
for i in range(num_samples):
# get next batch
l_sent = l_sentences[i]
r_sent = r_sentences[i]
sent = mx.nd.concat(l_sent, r_sent, dim=0)
l_len = len(l_sent)
l_tree = l_trees[i]
r_tree = r_trees[i]
inputs.append((sent, l_len, l_tree, r_tree))
z_fold = net.fold_encode(fold, sent, l_len, l_tree, r_tree)
fold_preds.append(z_fold)
if (i + 1) % batch_size == 0 or (i + 1) == num_samples:
fold_outs = fold([fold_preds])[0]
outs = mx.nd.concat(*[
net(sent, l_len, l_tree, r_tree)
for sent, l_len, l_tree, r_tree in inputs
],
dim=0)
if not almost_equal(fold_outs.asnumpy(), outs.asnumpy()):
print(fold_preds)
print('l_sents: ', l_sent, l_sentences[i - 1])
print('r_sents: ', r_sent, r_sentences[i - 1])
print('\n'.join(
(str(l_tree), str_tree(l_tree), str(r_tree),
str_tree(r_tree), str(l_trees[i - 1]),
str_tree(l_trees[i - 1]), str(r_trees[i - 1]),
str_tree(r_trees[i - 1]), str(fold))))
assert_almost_equal(fold_outs.asnumpy(), outs.asnumpy())
fold_preds = []
inputs = []
fold.reset()
def test(ctx, data_iter, best, mode='validation', num_iter=-1):
data_iter.reset()
num_samples = len(data_iter)
data_iter.set_context(ctx[0])
preds = []
labels = [mx.nd.array(data_iter.labels, ctx=ctx[0]).reshape((-1, 1))]
if opt.fold:
fold = Fold()
fold_preds = []
for j in tqdm(range(num_samples),
desc='Testing in {} mode'.format(mode)):
# get next batch
l_tree, l_sent, r_tree, r_sent, label = data_iter.next()
# forward calculation. the output is log probability
z = net.fold_encode(fold, l_sent, r_sent, l_tree, r_tree)
fold_preds.append(z)
# update weight after every batch_size samples, or when reaches last sample
if (j + 1) % batch_size == 0 or (j + 1) == num_samples:
preds.append(fold([fold_preds], True)[0])
fold_preds = []
else:
for j in tqdm(range(num_samples),
desc='Testing in {} mode'.format(mode)):
l_tree, l_sent, r_tree, r_sent, label = data_iter.next()
z = net(l_sent, r_sent, l_tree, r_tree)
preds.append(z)
if mode == 'validation' and num_iter >= 0:
preds = to_score(mx.nd.concat(*preds, dim=0))
metric.update(preds, labels)
names, values = metric.get()
metric.reset()
for name, acc in zip(names, values):
logging.info(mode + ' acc: %s=%f' % (name, acc))
if name == 'pearsonr':
test_r = acc
if test_r >= best:
best = test_r
logging.info('New optimum found: {}. Checkpointing.'.format(best))
net.collect_params().save(
'childsum_tree_lstm_{}.params'.format(num_iter))
test(ctx, test_iter, -1, 'test')
return best
def build_folds(data_spec=None, kfolds=None):
"""
Create Datasources, ML Model and Evaluation for each fold. Returns
a list of newly created evaluation IDs for all folds.
Args:
data_spec: the named tuple object that wraps dataset related
parameters.
kfolds: the integer number representing the number of folds.
Returns:
a list of newly created evaluation IDs.
"""
folds = [
Fold(data_spec=data_spec, this_fold=i, kfolds=kfolds)
for i in range(kfolds)
]
for f in folds:
f.build() # each fold creates entities
logger.info(f) # prints details of folds
return [f.ev_id for f in folds] # return list of eval IDs
def split_into_folds(self):
"""
Split each person's
images into each fold
"""
for x in range(self.num_folds):
images = []
labels = []
for person in self.people:
portion = len(person) // (self.num_folds - x)
images += person.pictures[0:portion]
labels += [person.name] * portion
del person.pictures[0:portion]
if (len(images) >= self.num_folds) and (len(labels)
== len(images)):
fold = Fold(images, labels)
self.folds.append(fold)
else:
raise ValueError
print('done')
def build_folds(data_spec=None, kfolds=None):
"""
Create Fold objects that will build Datasources for each fold.
Args:
data_spec: the named tuple object that wraps dataset related
parameters.
kfolds: the integer number representing the number of folds.
Returns:
a list of newly created Fold objects.
"""
folds = [
Fold(data_spec=data_spec, this_fold=i, kfolds=kfolds)
for i in range(kfolds)
]
for f in folds:
f.build() # each fold creates a Datasource
logger.debug(f)
return folds
def train(epoch, ctx, train_data, dev_data):
# initialization with context
if isinstance(ctx, mx.Context):
ctx = [ctx]
train_data.set_context(ctx[0])
dev_data.set_context(ctx[0])
# set up trainer for optimizing the network.
trainer = gluon.Trainer(net.collect_params(), optimizer, {
'learning_rate': opt.lr,
'wd': opt.wd
})
best_r = -1
Loss = gluon.loss.KLDivLoss()
fold = Fold()
for i in range(epoch):
train_data.reset()
num_samples = len(train_data)
# collect predictions and labels for evaluation metrics
preds = []
fold_preds = []
labels = [mx.nd.array(train_data.labels, ctx=ctx[0]).reshape((-1, 1))]
losses = []
if opt.fold:
for j in tqdm(range(num_samples),
desc='Training epoch {}'.format(i)):
# get next batch
l_tree, l_sent, r_tree, r_sent, label = train_data.next()
with ag.record():
# forward calculation. the output is log probability
z = net.fold_encode(fold, l_sent, r_sent, l_tree, r_tree)
fold_preds.append(z)
# calculate loss
with l_sent.context:
label = to_target(label)
loss = fold.record(0, Loss, z, label)
losses.append(loss)
# update weight after every batch_size samples, or when reaches last sample
if (j + 1) % batch_size == 0 or (j + 1) == num_samples:
with ag.record():
fold_preds, loss = fold([fold_preds, losses], True)
preds.append(fold_preds)
losses = []
fold_preds = []
fold.reset()
loss.backward()
trainer.step(batch_size)
else:
for j in tqdm(range(num_samples),
desc='Training epoch {}'.format(i)):
# get next batch
l_tree, l_sent, r_tree, r_sent, label = train_data.next()
# use autograd to record the forward calculation
with ag.record():
# forward calculation. the output is log probability
z = net(l_sent, r_sent, l_tree, r_tree)
# calculate loss
with l_sent.context:
label = to_target(label)
loss = Loss(z, label)
# backward calculation for gradients.
loss.backward()
preds.append(z)
# update weight after every batch_size samples
if (j + 1) % batch_size == 0 or (j + 1) == num_samples:
trainer.step(batch_size)
# translate log-probability to scores, and evaluate
preds = to_score(mx.nd.concat(*preds, dim=0))
metric.update(preds, labels)
names, values = metric.get()
metric.reset()
for name, acc in zip(names, values):
logging.info('training acc at epoch %d: %s=%f' % (i, name, acc))
best_r = test(ctx, dev_data, best_r, num_iter=i)