def main(args):
try:
if not os.path.isfile(args.checkpoint):
print('ERROR: Checkpoint file "%s" not found' % args.checkpoint)
print(
'Maybe you forgot to download pretraind models? Try running:')
print('bash scripts/download_models.sh')
return
if not os.path.isdir(args.output_dir):
print('Output directory "%s" does not exist; creating it' %
args.output_dir)
os.makedirs(args.output_dir)
if args.device == 'cpu':
device = torch.device('cpu')
elif args.device == 'gpu':
device = torch.device('cuda:0')
if not torch.cuda.is_available():
print('WARNING: CUDA not available; falling back to CPU')
device = torch.device('cpu')
# Load the model, with a bit of care in case there are no GPUs
map_location = 'cpu' if device == torch.device('cpu') else None
checkpoint = torch.load(args.checkpoint, map_location=map_location)
model = Sg2ImModel(**checkpoint['model_kwargs'])
model.load_state_dict(checkpoint['model_state'])
model.eval()
model.to(device)
# Load the scene graphs
scene_graphs = args.scene_graphs_json
# with open(args.scene_graphs_json, 'r') as f:
# scene_graphs = json.load(f)
print(type(scene_graphs))
print('Loaded graph!')
# Run the model forward
with torch.no_grad():
imgs, boxes_pred, masks_pred, _ = model.forward_json(scene_graphs)
imgs = imagenet_deprocess_batch(imgs)
# Save the generated images
for i in range(imgs.shape[0]):
img_np = imgs[i].numpy().transpose(1, 2, 0)
img_path = os.path.join(args.output_dir, 'img' + args.id + '.png')
imwrite(img_path, img_np)
print('Drawing now!')
# Draw the scene graphs
if args.draw_scene_graphs == 1:
for i, sg in enumerate(scene_graphs):
sg_img = vis.draw_scene_graph(sg['objects'],
sg['relationships'])
sg_img_path = os.path.join(args.output_dir,
'sg' + args.id + '.png' % i)
imwrite(sg_img_path, sg_img)
return True
except ():
return False
def main(args):
if not os.path.isfile(args.checkpoint):
print('ERROR: Checkpoint file "%s" not found' % args.checkpoint)
print('Maybe you forgot to download pretraind models? Try running:')
print('bash scripts/download_models.sh')
return
if not os.path.isdir(args.output_dir):
print('Output directory "%s" does not exist; creating it' % args.output_dir)
os.makedirs(args.output_dir)
if args.device == 'cpu':
device = torch.device('cpu')
elif args.device == 'gpu':
device = torch.device('cuda:0')
if not torch.cuda.is_available():
print('WARNING: CUDA not available; falling back to CPU')
device = torch.device('cpu')
# Load the model, with a bit of care in case there are no GPUs
map_location = 'cpu' if device == torch.device('cpu') else None
checkpoint = torch.load(args.checkpoint, map_location=map_location)
model = Sg2ImModel(**checkpoint['model_kwargs'])
model.load_state_dict(checkpoint['model_state'], strict=False)
model.eval()
model.to(device)
# Load the scene graphs
with open(args.scene_graphs_json, 'r') as f:
scene_graphs = json.load(f)
# Run the model forward
with torch.no_grad():
# imgs, boxes_pred, masks_pred, _ = model.forward_json(scene_graphs)
imgs, boxes_pred, masks_pred, objs, layout, layout_boxes_t, layout_masks, obj_to_img, sg_context_pred, _, _ = model.forward_json(scene_graphs)
imgs = imagenet_deprocess_batch(imgs)
layout_boxes = layout_boxes_t.numpy()
np_imgs = []
# Save the generated images
import numpy as np
for i in range(imgs.shape[0]):
# img_np = imgs[i].numpy().transpose(1, 2, 0)
img_np = (imgs[i].numpy().transpose(1, 2, 0) * 255.0).astype(np.uint8)
img_path = os.path.join(args.output_dir, 'img%06d.png' % i)
imwrite(img_path, img_np)
np_imgs.append(img_np)
# Draw the scene graphs
if args.draw_scene_graphs == 1:
for i, sg in enumerate(scene_graphs):
sg_img = vis.draw_scene_graph(sg['objects'], sg['relationships'])
sg_img_path = os.path.join(args.output_dir, 'sg%06d.png' % i)
imwrite(sg_img_path, sg_img)
def draw_results(model, output_dir, scene_graphs):
if not os.path.exists(output_dir):
os.makedirs(output_dir)
all_objs, all_triplets, all_obj_to_img = encode_scene_graphs_list(model, scene_graphs)
# Run the model forward
with torch.no_grad():
imgs_pred, boxes_pred, _, _, obj_to_img = model(all_objs, all_triplets, all_obj_to_img)
imgs_pred = deprocess_batch(imgs_pred)
boxes_pred = boxes_pred.cpu()
obj_to_img = obj_to_img.cpu()
# Save the generated images
draw_predictions(scene_graphs, imgs_pred, boxes_pred, obj_to_img, output_dir)
for i, sg in enumerate(scene_graphs):
sg_img = vis.draw_scene_graph(sg['objects'], sg['relationships'])
sg_img_path = os.path.join(output_dir, 'img_%06d_sg.png' % i)
imwrite(sg_img_path, sg_img)
plt.close("all")
def draw_scene_graphs(scene_graphs, output_dir, vocab):
for i, sg in enumerate(scene_graphs):
sg_img = vis.draw_scene_graph(sg['objects'], sg['relationships'], vocab)
sg_img_path = os.path.join(output_dir, 'img_%06d_sg.png' % i)
imwrite(sg_img_path, sg_img)
def main(args):
if not os.path.isfile(args.checkpoint):
print('ERROR: Checkpoint file "%s" not found' % args.checkpoint)
print('Maybe you forgot to download pretraind models? Try running:')
print('bash scripts/download_models.sh')
return
if not os.path.isdir(args.output_dir):
print('Output directory "%s" does not exist; creating it' %
args.output_dir)
os.makedirs(args.output_dir)
if args.device == 'cpu':
device = torch.device('cpu')
elif args.device == 'gpu':
device = torch.device('cuda:0')
if not torch.cuda.is_available():
print('WARNING: CUDA not available; falling back to CPU')
device = torch.device('cpu')
# Load the model, with a bit of care in case there are no GPUs
map_location = 'cpu' if device == torch.device('cpu') else None
checkpoint = torch.load(args.checkpoint, map_location=map_location)
model = Sg2ImModel(**checkpoint['model_kwargs'])
model.load_state_dict(checkpoint['model_state'])
model.eval()
model.to(device)
SCENE_GRAPH_DIR = args.scene_graph_dir
scene_graphs = []
# Load the scene graphs
for filename in os.listdir(SCENE_GRAPH_DIR):
print("opening file: {}".format(filename))
with open(os.path.join(SCENE_GRAPH_DIR, filename), 'r') as f:
sg = json.load(f)
scene_graphs.append(sg)
for sg_idx, sg in enumerate(scene_graphs):
# Run the model forward
with torch.no_grad():
try:
imgs, boxes_pred, masks_pred, _ = model.forward_json(sg)
except ValueError as err:
print("ValueError: {}".format(err))
continue
imgs = imagenet_deprocess_batch(imgs)
# Save the generated images
for i in range(imgs.shape[0]):
img_np = imgs[i].numpy().transpose(1, 2, 0)
img_path = os.path.join(args.output_dir, 'img%06d.png' % sg_idx)
imwrite(img_path, img_np)
# Draw the scene graphs
if args.draw_scene_graphs == 1:
for i, sg_ in enumerate(sg):
sg_img = vis.draw_scene_graph(sg_['objects'],
sg_['relationships'])
sg_img_path = os.path.join(args.output_dir,
'sg%06d.png' % sg_idx)
imwrite(sg_img_path, sg_img)
def run_model(args, checkpoint, output_dir, loader=None):
vocab = checkpoint['model_kwargs']['vocab']
model = build_model(args, checkpoint)
if loader is None:
loader = build_loader(args, checkpoint)
img_dir = makedir(output_dir, 'images')
graph_dir = makedir(output_dir, 'graphs', args.save_graphs)
gt_img_dir = makedir(output_dir, 'images_gt', args.save_gt_imgs)
data_path = os.path.join(output_dir, 'data.pt')
data = {
'vocab': vocab,
'objs': [],
'masks_pred': [],
'boxes_pred': [],
'masks_gt': [],
'boxes_gt': [],
'filenames': [],
}
if args.model == 'bm_FH':
FH_dir_train = args.FH_dir_train
FH_dir_val = args.FH_dir_val
if args.model == 'bm_FHrec':
FH_dir_train = args.FHrec_dir_train
FH_dir_val = args.FHrec_dir_val
if args.model == 'bm_FH64':
FH_dir_train = args.FH64_dir_train
FH_dir_val = args.FH64_dir_val
if args.model == 'bm_FHrec64':
FH_dir_train = args.FHrec64_dir_train
FH_dir_val = args.FHrec64_dir_val
FH_objs_train, FH_edges_train, IDs_train = torch.load(FH_dir_train)
FH_objs_val, FH_edges_val, IDs_val = torch.load(FH_dir_val)
IDs_train = torch.tensor(IDs_train)
IDs_val = torch.tensor(IDs_val)
if args.which_data == 'train':
IDs = IDs_train
FH_objs = FH_objs_train
FH_edges = FH_edges_train
else:
IDs = IDs_val
FH_objs = FH_objs_val
FH_edges = FH_edges_val
img_idx = 0
for batch in loader:
masks = None
if len(batch) == 6:
imgs, objs, boxes, triples, obj_to_img, triple_to_img = [x.cuda() for x in batch]
elif len(batch) == 7:
imgs, objs, boxes, masks, triples, obj_to_img, triple_to_img = [x.cuda() for x in batch]
imgs, objs, boxes, masks, triples, obj_to_img, triple_to_img, imgs_ids = [x.cuda() for x in batch]
# get FH by images within a batch
fh_obj, fh_edge = [],[]
for i in range(imgs_ids.shape[0]):
idd = ((IDs == imgs_ids[i].item()).nonzero())
fh_obj_i = FH_objs[idd]
fh_obj.append(fh_obj_i)
fh_edge_i = FH_edges[idd]
fh_edge.append(fh_edge_i)
fh_obj = torch.cat(fh_obj)
fh_edge = torch.cat(fh_edge)
imgs_gt = imagenet_deprocess_batch(imgs)
boxes_gt = None
masks_gt = None
if args.use_gt_boxes:
boxes_gt = boxes
if args.use_gt_masks:
masks_gt = masks
# Run the model with predicted masks
model_out = model(objs, triples, fh_obj, fh_edge, obj_to_img,
boxes_gt=boxes_gt, masks_gt=masks_gt)
# model_out = model(objs, triples, obj_to_img,
# boxes_gt=boxes_gt, masks_gt=masks_gt)
imgs_pred, boxes_pred, masks_pred, _ = model_out
imgs_pred = imagenet_deprocess_batch(imgs_pred)
obj_data = [objs, boxes_pred, masks_pred]
_, obj_data = split_graph_batch(triples, obj_data, obj_to_img,
triple_to_img)
objs, boxes_pred, masks_pred = obj_data
obj_data_gt = [boxes.data]
if masks is not None:
obj_data_gt.append(masks.data)
triples, obj_data_gt = split_graph_batch(triples, obj_data_gt,
obj_to_img, triple_to_img)
boxes_gt, masks_gt = obj_data_gt[0], None
if masks is not None:
masks_gt = obj_data_gt[1]
for i in range(imgs_pred.size(0)):
img_filename = '%04d.png' % img_idx
if args.save_gt_imgs:
img_gt = imgs_gt[i].numpy().transpose(1, 2, 0)
img_gt_path = os.path.join(gt_img_dir, img_filename)
imsave(img_gt_path, img_gt)
img_pred = imgs_pred[i]
img_pred_np = imgs_pred[i].numpy().transpose(1, 2, 0)
img_path = os.path.join(img_dir, img_filename)
imsave(img_path, img_pred_np)
data['objs'].append(objs[i].cpu().clone())
data['masks_pred'].append(masks_pred[i].cpu().clone())
data['boxes_pred'].append(boxes_pred[i].cpu().clone())
data['boxes_gt'].append(boxes_gt[i].cpu().clone())
data['filenames'].append(img_filename)
cur_masks_gt = None
if masks_gt is not None:
cur_masks_gt = masks_gt[i].cpu().clone()
data['masks_gt'].append(cur_masks_gt)
if args.save_graphs:
graph_img = draw_scene_graph(vocab, objs[i], triples[i])
graph_path = os.path.join(graph_dir, img_filename)
imsave(graph_path, graph_img)
img_idx += 1
torch.save(data, data_path)
print('Saved %d images' % img_idx)
debug=True,
max_objects=5,
dense_scenes=False)
# ss = 0
# ii = []
# for scene in dset.clevr_data['scenes']:
# ss += len(scene['objects'])
# ii.append(len(scene['objects']))
# ss /= len(dset.clevr_data['scenes'])
#
# import matplotlib.pyplot as plt
#
# ii = np.array(ii)
# _ = plt.hist(ii, bins='auto') # arguments are passed to np.histogram
# plt.savefig("hist.png")
it = dset[2]
while True:
# idx = 5149
idx = np.random.randint(0, len(dset))
item = dset[idx]
image, objs, boxes, triplets = item
image = deprocess_batch(torch.unsqueeze(image, 0))[0]
cv2.imwrite('/tmp/img.png', np.transpose(image.cpu().numpy(), [1, 2, 0]))
draw_item(item, image_size=dset.image_size) # dset.clevr_data['scenes'][index]
plt.figure()
plt.imshow(draw_scene_graph(dset.clevr_data['scenes'][idx]['objects'],
triplets=dset.clevr_data['scenes'][idx]['relationships'], vocab=dset.vocab))
plt.savefig('/tmp/sg.png')
def run_model(args, checkpoint, output_dir, loader=None):
vocab = checkpoint['model_kwargs']['vocab']
model = build_model(args, checkpoint)
if loader is None:
loader = build_loader(args, checkpoint)
img_dir = makedir(output_dir, 'images')
graph_dir = makedir(output_dir, 'graphs', args.save_graphs)
gt_img_dir = makedir(output_dir, 'images_gt', args.save_gt_imgs)
data_path = os.path.join(output_dir, 'data.pt')
data = {
'vocab': vocab,
'objs': [],
'masks_pred': [],
'boxes_pred': [],
'masks_gt': [],
'boxes_gt': [],
'filenames': [],
}
img_idx = 0
for batch in loader:
masks = None
if len(batch) == 6:
imgs, objs, boxes, triples, obj_to_img, triple_to_img = [
x.cuda() for x in batch
]
elif len(batch) == 7:
imgs, objs, boxes, masks, triples, obj_to_img, triple_to_img = [
x.cuda() for x in batch
]
imgs_gt = imagenet_deprocess_batch(imgs)
boxes_gt = None
masks_gt = None
if args.use_gt_boxes:
boxes_gt = boxes
if args.use_gt_masks:
masks_gt = masks
# Run the model with predicted masks
model_out = model(objs,
triples,
obj_to_img,
boxes_gt=boxes_gt,
masks_gt=masks_gt)
imgs_pred, boxes_pred, masks_pred, _ = model_out
imgs_pred = imagenet_deprocess_batch(imgs_pred)
obj_data = [objs, boxes_pred, masks_pred]
_, obj_data = split_graph_batch(triples, obj_data, obj_to_img,
triple_to_img)
objs, boxes_pred, masks_pred = obj_data
obj_data_gt = [boxes.data]
if masks is not None:
obj_data_gt.append(masks.data)
triples, obj_data_gt = split_graph_batch(triples, obj_data_gt,
obj_to_img, triple_to_img)
boxes_gt, masks_gt = obj_data_gt[0], None
if masks is not None:
masks_gt = obj_data_gt[1]
for i in range(imgs_pred.size(0)):
img_filename = '%04d.png' % img_idx
if args.save_gt_imgs:
img_gt = imgs_gt[i].numpy().transpose(1, 2, 0)
img_gt_path = os.path.join(gt_img_dir, img_filename)
imsave(img_gt_path, img_gt)
img_pred = imgs_pred[i]
img_pred_np = imgs_pred[i].numpy().transpose(1, 2, 0)
img_path = os.path.join(img_dir, img_filename)
imsave(img_path, img_pred_np)
data['objs'].append(objs[i].cpu().clone())
data['masks_pred'].append(masks_pred[i].cpu().clone())
data['boxes_pred'].append(boxes_pred[i].cpu().clone())
data['boxes_gt'].append(boxes_gt[i].cpu().clone())
data['filenames'].append(img_filename)
cur_masks_gt = None
if masks_gt is not None:
cur_masks_gt = masks_gt[i].cpu().clone()
data['masks_gt'].append(cur_masks_gt)
if args.save_graphs:
graph_img = draw_scene_graph(vocab, objs[i], triples[i])
graph_path = os.path.join(graph_dir, img_filename)
imsave(graph_path, graph_img)
img_idx += 1
torch.save(data, data_path)
print('Saved %d images' % img_idx)