assert (display.shape == (1, V))
output_str = str(k) + ": "
for v in xrange(self._V):
if display[:, v]:
output_str += vocab[v] + "\t"
print(output_str)
"""
run HDP on a synthetic corpus.
"""
if __name__ == '__main__':
from util.vocabulary import Vocabulary, parse_raw_text
bdir = "../../data/nips12/"
#data = import_monolingual_data(bdir + "doc.dat")
data = parse_raw_text(bdir)
data = data[:50]
voca = Vocabulary(exclude_stopwords=True)
corpus = [voca.doc2bow(doc) for doc in data]
#corpus = numpy.array([numpy.random.poisson(1, k) for k in numpy.random.poisson(50, 20)])
corpus = dict((i, j) for i, j in enumerate(corpus))
gs = UncollapsedGibbsSampling()
gs._initialize(corpus)
gs.sample(100)
print(gs._K)
print(gs._n_kv)
def __init__(self, config, overfit=False):
super().__init__()
self.que_vocabulary = Vocabulary(config["dataset"]["word2id_path"])
self.image_features = []
self.image_boxes = []
self.image_labels = []
self.image_captions = []
self.questions = []
self.questions_ids = []
self.questions_vecs = []
self.answers = []
self.config = config
with open(config['dataset']['test']['test_qids'], 'r') as f:
self.test_qids = json.load(f)
with open(config['dataset']['test']['test_questions'], 'r') as f:
self.test_questions = json.load(f)
with open(config['dataset']['test']['test_answers'], 'r') as f:
self.test_answers = json.load(f)
with open(config['dataset']['test']['test_gt_facts'], 'r') as f:
self.test_gt_facts = json.load(f)
with open(config['dataset']['test']['test_captions'], 'r') as f:
self.test_captions = json.load(f)
with open(config['dataset']['test']['test_semantic_graph'], 'rb') as f:
self.semantic_graphs = pickle.load(f, encoding='iso-8859-1')
with open(config['dataset']['test']['test_labels'], 'r') as f:
self.test_labels = json.load(f)
with open(config['dataset']['test']['test_whs'], 'r') as f:
self.test_whs = json.load(f)
with open(config['dataset']['test']['test_top100_facts_graph'],
'rb') as f:
self.test_top_facts = pickle.load(f, encoding='iso-8859-1')
self.test_features = np.load(
config['dataset']['test']['test_features'])
self.test_bboxes = np.load(config['dataset']['test']['test_bboxes'])
if overfit:
self.test_qids = self.test_qids[:100]
self.test_questions = self.test_questions[:100]
self.test_answers = self.test_answers[:100]
self.test_gt_facts = self.test_gt_facts[:100]
self.test_top_facts = self.test_top_facts[:100]
self.test_captions = self.test_captions[:100]
self.test_bboxes = self.test_bboxes[:100]
self.test_features = self.test_features[:100]
self.test_whs = self.test_whs[:100]
self.semantic_graphs = self.semantic_graphs[:100]
def train():
parser = argparse.ArgumentParser()
# 配置文件
parser.add_argument(
"--config-yml",
default="exp_fvqa/exp2.yml",
help=
"Path to a config file listing reader, model and solver parameters.")
parser.add_argument("--cpu-workers",
type=int,
default=8,
help="Number of CPU workers for dataloader.")
parser.add_argument(
"--save-dirpath",
default="fvqa/exp_data/checkpoints",
help=
"Path of directory to create checkpoint directory and save checkpoints."
)
parser.add_argument(
"--load-pthpath",
default="",
help="To continue training, path to .pth file of saved checkpoint.")
parser.add_argument("--gpus", default="", help="gpus")
parser.add_argument(
"--overfit",
action="store_true",
help="Whether to validate on val split after every epoch.")
parser.add_argument(
"--validate",
action="store_true",
help="Whether to validate on val split after every epoch.")
args = parser.parse_args()
# set mannual seed
torch.manual_seed(10)
torch.cuda.manual_seed(10)
cudnn.benchmark = True
cudnn.deterministic = True
config = yaml.load(open(args.config_yml))
device = torch.device("cuda:0") if args.gpus != "cpu" else torch.device(
"cpu")
# Print config and args.
print(yaml.dump(config, default_flow_style=False))
for arg in vars(args):
print("{:<20}: {}".format(arg, getattr(args, arg)))
print('Loading TrainDataset...')
train_dataset = FvqaTrainDataset(config, overfit=args.overfit)
train_dataloader = DataLoader(train_dataset,
batch_size=config['solver']['batch_size'],
num_workers=args.cpu_workers,
shuffle=True,
collate_fn=collate_fn)
if args.validate:
print('Loading TestDataset...')
val_dataset = FvqaTestDataset(config, overfit=args.overfit)
val_dataloader = DataLoader(val_dataset,
batch_size=config['solver']['batch_size'],
num_workers=args.cpu_workers,
shuffle=True,
collate_fn=collate_fn)
print('Loading glove...')
que_vocab = Vocabulary(config['dataset']['word2id_path'])
glove = np.load(config['dataset']['glove_vec_path'])
glove = torch.Tensor(glove)
print('Building Model...')
model = CMGCNnet(config,
que_vocabulary=que_vocab,
glove=glove,
device=device)
if torch.cuda.device_count() > 1 and args.gpus != "cpu":
print("Let's use", torch.cuda.device_count(), "GPUs!")
model = nn.DataParallel(model)
model = model.to(device)
print(model)
iterations = len(train_dataset) // config["solver"]["batch_size"] + 1
def lr_lambda_fun(current_iteration: int) -> float:
current_epoch = float(current_iteration) / iterations
if current_epoch <= config["solver"]["warmup_epochs"]:
alpha = current_epoch / float(config["solver"]["warmup_epochs"])
return config["solver"]["warmup_factor"] * (1. - alpha) + alpha
else:
idx = bisect(config["solver"]["lr_milestones"], current_epoch)
return pow(config["solver"]["lr_gamma"], idx)
optimizer = optim.Adamax(model.parameters(),
lr=config["solver"]["initial_lr"])
scheduler = lr_scheduler.LambdaLR(optimizer, lr_lambda=lr_lambda_fun)
T = iterations * (config["solver"]["num_epochs"] -
config["solver"]["warmup_epochs"] + 1)
scheduler2 = lr_scheduler.CosineAnnealingLR(
optimizer, int(T), eta_min=config["solver"]["eta_min"], last_epoch=-1)
summary_writer = SummaryWriter(log_dir=args.save_dirpath)
checkpoint_manager = CheckpointManager(model,
optimizer,
args.save_dirpath,
config=config)
if args.load_pthpath == "":
start_epoch = 0
else:
start_epoch = int(args.load_pthpath.split("_")[-1][:-4])
model_state_dict, optimizer_state_dict = load_checkpoint(
args.load_pthpath)
if isinstance(model, nn.DataParallel):
model.module.load_state_dict(model_state_dict)
else:
model.load_state_dict(model_state_dict)
optimizer.load_state_dict(optimizer_state_dict)
print("Loading resume model from {}...".format(args.load_pthpath))
global_iteration_step = start_epoch * iterations
for epoch in range(start_epoch, config['solver']['num_epochs']):
print(f"\nTraining for epoch {epoch}:")
train_answers = []
train_preds = []
for i, batch in enumerate(tqdm(train_dataloader)):
optimizer.zero_grad()
fact_batch_graph = model(batch)
batch_loss = cal_batch_loss(fact_batch_graph,
batch,
device,
neg_weight=0.1,
pos_weight=0.9)
batch_loss.backward()
optimizer.step()
fact_graphs = dgl.unbatch(fact_batch_graph)
for i, fact_graph in enumerate(fact_graphs):
train_pred = fact_graph.ndata['h'].squeeze() # (num_nodes,1)
train_preds.append(train_pred) # [(num_nodes,)]
train_answers.append(batch['facts_answer_id_list'][i])
summary_writer.add_scalar('train/loss', batch_loss,
global_iteration_step)
summary_writer.add_scalar("train/lr",
optimizer.param_groups[0]["lr"],
global_iteration_step)
summary_writer.add_text('train/loss', str(batch_loss.item()),
global_iteration_step)
summary_writer.add_text('train/lr',
str(optimizer.param_groups[0]["lr"]),
global_iteration_step)
if global_iteration_step <= iterations * config["solver"][
"warmup_epochs"]:
scheduler.step(global_iteration_step)
else:
global_iteration_step_in_2 = iterations * config["solver"][
"warmup_epochs"] + 1 - global_iteration_step
scheduler2.step(int(global_iteration_step_in_2))
global_iteration_step = global_iteration_step + 1
torch.cuda.empty_cache()
checkpoint_manager.step()
train_acc_1, train_acc_3 = cal_acc(
train_answers, train_preds)
print(
"trainacc@1={:.2%} & trainacc@3={:.2%} "
.format(train_acc_1, train_acc_3))
summary_writer.add_scalars(
'train/acc', {
'acc@1': train_acc_1,
'acc@3': train_acc_3
}, epoch)
if args.validate:
model.eval()
answers = [] # [batch_answers,...]
preds = [] # [batch_preds,...]
print(f"\nValidation after epoch {epoch}:")
for i, batch in enumerate(tqdm(val_dataloader)):
with torch.no_grad():
fact_batch_graph = model(batch)
batch_loss = cal_batch_loss(fact_batch_graph,
batch,
device,
neg_weight=0.1,
pos_weight=0.9)
summary_writer.add_scalar('test/loss', batch_loss, epoch)
fact_graphs = dgl.unbatch(fact_batch_graph)
for i, fact_graph in enumerate(fact_graphs):
pred = fact_graph.ndata['h'].squeeze() # (num_nodes,1)
preds.append(pred) # [(num_nodes,)]
answers.append(batch['facts_answer_id_list'][i])
acc_1, acc_3 = cal_acc(answers, preds)
print("acc@1={:.2%} & acc@3={:.2%} ".
format(acc_1, acc_3))
summary_writer.add_scalars('test/acc', {
'acc@1': acc_1,
'acc@3': acc_3
}, epoch)
model.train()
torch.cuda.empty_cache()
print('Train finished !!!')
summary_writer.close()
def __init__(self, config, overfit=False):
super().__init__()
self.que_vocabulary = Vocabulary(config["dataset"]["word2id_path"])
self.image_features = []
self.image_boxes = []
self.image_labels = []
self.image_captions = []
self.questions = []
self.questions_ids = []
self.questions_vecs = []
self.answers = []
self.config = config
# 问题的 id
with open(config['dataset']['train']['train_qids'], 'r') as f:
self.train_qids = json.load(f)
# 问题
with open(config['dataset']['train']['train_questions'], 'r') as f:
self.train_questions = json.load(f)
# 答案
with open(config['dataset']['train']['train_answers'], 'r') as f:
self.train_answers = json.load(f)
# gt fact
with open(config['dataset']['train']['train_gt_facts'], 'r') as f:
self.train_gt_facts = json.load(f)
# caption
with open(config['dataset']['train']['train_captions'], 'r') as f:
self.train_captions = json.load(f)
# semantic graph
with open(config['dataset']['train']['train_semantic_graph'], 'rb') as f:
self.semantic_graphs = pickle.load(f, encoding='iso-8859-1')
# 图像bbox 对应的 label
with open(config['dataset']['train']['train_labels'], 'r') as f:
self.train_labels = json.load(f)
# 图像的长宽
with open(config['dataset']['train']['train_whs'], 'r') as f:
self.train_whs = json.load(f)
# 抽取到的 facts
# with open(config['dataset']['train']['train_facts_graph'], 'rb') as f:
# self.train_facts = pickle.load(f, encoding='iso-8859-1')
with open(config['dataset']['train']['train100_facts_graph'],
'rb') as f:
self.train_top_facts = pickle.load(f, encoding='iso-8859-1')
# 图像 bbox 的特征
self.train_features = np.load(
config['dataset']['train']['train_features'])
# 图像的 bbox 几何信息
self.train_bboxes = np.load(config['dataset']['train']['train_bboxes'])
if overfit:
self.train_qids = self.train_qids[:100]
self.train_questions = self.train_questions[:100]
self.train_answers = self.train_answers[:100]
self.train_gt_facts = self.train_gt_facts[:100]
# self.train_facts = self.train_facts[:100]
self.train_top_facts = self.train_top_facts[:100]
self.train_captions = self.train_captions[:100]
self.train_bboxes = self.train_bboxes[:100]
self.train_features = self.train_features[:100]
self.train_whs = self.train_whs[:100]
self.semantic_graphs=self.semantic_graphs[:100]
