def create_cpu_pool(no_thread, use_process, maxtasksperchild=1000):
if no_thread == 0:
return None
elif use_process:
cpu_pool = Pool(no_thread, maxtasksperchild=maxtasksperchild)
else:
cpu_pool = ThreadPool(no_thread)
cpu_pool._maxtasksperchild = maxtasksperchild
return cpu_pool
ann_file = os.path.join(args.data_dir, 'mscoco_val{}_annotations.json'.format(args.year))
vqa_eval_listener = VQAEvaluator(tokenizer, dump_file, ann_file, ques_file, require=networks[0].prediction)
# start actual training
best_val_acc, best_train_acc = 0, 0
for t in range(start_epoch, no_epoch):
# CPU/GPU option
# h5 requires a Tread pool while raw images are more efficient with processes
if require_multiprocess:
cpu_pool = Pool(args.no_thread, maxtasksperchild=1000)
else:
cpu_pool = ThreadPool(args.no_thread)
cpu_pool._maxtasksperchild = 1000
logger.info('Epoch {}/{}..'.format(t + 1,no_epoch))
train_iterator = Iterator(trainset,
batch_size=batch_size,
batchifier=train_batchifier,
shuffle=True,
pool=cpu_pool)
[train_loss, train_accuracy] = train_evaluator.process(sess, train_iterator, outputs=outputs + [optimizer])
valid_loss, valid_accuracy = 0,0
if not merge_dataset:
valid_iterator = Iterator(validset,
