def create_cpu_pool(no_thread, use_process, maxtasksperchild=1000): if no_thread == 0: return None elif use_process: cpu_pool = Pool(no_thread, maxtasksperchild=maxtasksperchild) else: cpu_pool = ThreadPool(no_thread) cpu_pool._maxtasksperchild = maxtasksperchild return cpu_pool
ann_file = os.path.join(args.data_dir, 'mscoco_val{}_annotations.json'.format(args.year)) vqa_eval_listener = VQAEvaluator(tokenizer, dump_file, ann_file, ques_file, require=networks[0].prediction) # start actual training best_val_acc, best_train_acc = 0, 0 for t in range(start_epoch, no_epoch): # CPU/GPU option # h5 requires a Tread pool while raw images are more efficient with processes if require_multiprocess: cpu_pool = Pool(args.no_thread, maxtasksperchild=1000) else: cpu_pool = ThreadPool(args.no_thread) cpu_pool._maxtasksperchild = 1000 logger.info('Epoch {}/{}..'.format(t + 1,no_epoch)) train_iterator = Iterator(trainset, batch_size=batch_size, batchifier=train_batchifier, shuffle=True, pool=cpu_pool) [train_loss, train_accuracy] = train_evaluator.process(sess, train_iterator, outputs=outputs + [optimizer]) valid_loss, valid_accuracy = 0,0 if not merge_dataset: valid_iterator = Iterator(validset,