def get_baseline_model(args):
vocab = utils.load_vocab(args.vocab_json)
if args.baseline_start_from is not None:
model, kwargs = utils.load_baseline(args.baseline_start_from)
elif args.model_type == 'LSTM':
kwargs = {
'vocab': vocab,
'rnn_wordvec_dim': args.rnn_wordvec_dim,
'rnn_dim': args.rnn_hidden_dim,
'rnn_num_layers': args.rnn_num_layers,
'rnn_dropout': args.rnn_dropout,
'fc_dims': parse_int_list(args.classifier_fc_dims),
'fc_use_batchnorm': args.classifier_batchnorm == 1,
'fc_dropout': args.classifier_dropout,
}
model = LstmModel(**kwargs)
elif args.model_type == 'CNN+LSTM':
kwargs = {
'vocab': vocab,
'rnn_wordvec_dim': args.rnn_wordvec_dim,
'rnn_dim': args.rnn_hidden_dim,
'rnn_num_layers': args.rnn_num_layers,
'rnn_dropout': args.rnn_dropout,
'cnn_feat_dim': parse_int_list(args.feature_dim),
'cnn_num_res_blocks': args.cnn_num_res_blocks,
'cnn_res_block_dim': args.cnn_res_block_dim,
'cnn_proj_dim': args.cnn_proj_dim,
'cnn_pooling': args.cnn_pooling,
'fc_dims': parse_int_list(args.classifier_fc_dims),
'fc_use_batchnorm': args.classifier_batchnorm == 1,
'fc_dropout': args.classifier_dropout,
}
model = CnnLstmModel(**kwargs)
elif args.model_type == 'CNN+LSTM+SA':
kwargs = {
'vocab': vocab,
'rnn_wordvec_dim': args.rnn_wordvec_dim,
'rnn_dim': args.rnn_hidden_dim,
'rnn_num_layers': args.rnn_num_layers,
'rnn_dropout': args.rnn_dropout,
'cnn_feat_dim': parse_int_list(args.feature_dim),
'stacked_attn_dim': args.stacked_attn_dim,
'num_stacked_attn': args.num_stacked_attn,
'fc_dims': parse_int_list(args.classifier_fc_dims),
'fc_use_batchnorm': args.classifier_batchnorm == 1,
'fc_dropout': args.classifier_dropout,
}
model = CnnLstmSaModel(**kwargs)
if model.rnn.token_to_idx != vocab['question_token_to_idx']:
# Make sure new vocab is superset of old
for k, v in model.rnn.token_to_idx.items():
assert k in vocab['question_token_to_idx']
assert vocab['question_token_to_idx'][k] == v
for token, idx in vocab['question_token_to_idx'].items():
model.rnn.token_to_idx[token] = idx
kwargs['vocab'] = vocab
model.rnn.expand_vocab(vocab['question_token_to_idx'])
model.cuda()
model.train()
return model, kwargs
def main(args):
model = None
try:
with open(args.properties_json, 'r') as f:
properties = json.load(f)
except:
print ("Unable to open properties file (properties_json argument)")
exit()
if args.baseline_model is not None:
print('Loading baseline model from ', args.baseline_model)
model, _ = utils.load_baseline(args.baseline_model)
if args.vocab_json is not None:
new_vocab = utils.load_vocab(args.vocab_json)
model.rnn.expand_vocab(new_vocab['question_token_to_idx'])
elif args.program_generator is not None and args.execution_engine is not None:
print('Loading program generator from ', args.program_generator)
program_generator, _ = utils.load_program_generator(args.program_generator)
print('Loading execution engine from ', args.execution_engine)
execution_engine, _ = utils.load_execution_engine(args.execution_engine, verbose=False)
if args.vocab_json is not None:
new_vocab = utils.load_vocab(args.vocab_json)
program_generator.expand_encoder_vocab(new_vocab['question_token_to_idx'])
model = (program_generator, execution_engine)
else:
print('Must give either --baseline_model or --program_generator and --execution_engine')
return
call_model(args, model)
def main(args):
print()
model = None
import pdb
pdb.set_trace()
if args.baseline_model is not None:
print('Loading baseline model from ', args.baseline_model)
model, _ = utils.load_baseline(args.baseline_model)
if args.vocab_json is not None:
new_vocab = utils.load_vocab(args.vocab_json)
model.rnn.expand_vocab(new_vocab['question_token_to_idx'])
elif args.program_generator is not None and args.execution_engine is not None:
print('Loading program generator from ', args.program_generator)
program_generator, _ = utils.load_program_generator(
args.program_generator)
print('Loading execution engine from ', args.execution_engine)
execution_engine, _ = utils.load_execution_engine(
args.execution_engine, verbose=False)
if args.vocab_json is not None:
new_vocab = utils.load_vocab(args.vocab_json)
program_generator.expand_encoder_vocab(
new_vocab['question_token_to_idx'])
model = (program_generator, execution_engine)
else:
print(
'Must give either --baseline_model or --program_generator and --execution_engine'
)
return
if args.question is not None and args.image is not None:
run_single_example(args, model)
else:
vocab = load_vocab(args)
loader_kwargs = {
'question_h5': args.input_question_h5,
'feature_h5': args.input_features_h5,
'vocab': vocab,
'batch_size': args.batch_size,
}
if args.num_samples is not None and args.num_samples > 0:
loader_kwargs['max_samples'] = args.num_samples
if args.family_split_file is not None:
with open(args.family_split_file, 'r') as f:
loader_kwargs['question_families'] = json.load(f)
with ClevrDataLoader(**loader_kwargs) as loader:
run_batch(args, model, loader)
def init_model(args):
model = None
if args.baseline_model is not None:
print('Loading baseline model from ', args.baseline_model)
model, _ = utils.load_baseline(args.baseline_model)
if args.vocab_json is not None:
new_vocab = utils.load_vocab(args.vocab_json)
model.rnn.expand_vocab(new_vocab['question_token_to_idx'])
elif args.program_generator is not None and args.execution_engine is not None:
print('Loading program generator from ', args.program_generator)
program_generator, _ = utils.load_program_generator(args.program_generator)
print('Loading execution engine from ', args.execution_engine)
execution_engine, _ = utils.load_execution_engine(args.execution_engine, verbose=False)
if args.vocab_json is not None:
new_vocab = utils.load_vocab(args.vocab_json)
program_generator.expand_encoder_vocab(new_vocab['question_token_to_idx'])
model = (program_generator, execution_engine)
else:
print('Must give either --baseline_model or --program_generator and --execution_engine')
return None
return model
def main(args):
global AVAILABLE_OBJECTS, AVAILABLE_MATERIALS, AVAILABLE_SIZES, AVAILABLE_COLOURS
global NUM_AVAILABLE_OBJECTS, NUM_AVAILABLE_MATERIALS, NUM_AVAILABLE_SIZES, NUM_AVAILABLE_COLOURS
global obj_probs, material_probs, colour_probs, size_probs
global save_directory
model = None
try:
with open(args.properties_json, 'r') as f:
properties = json.load(f)
for name, rgb in properties['colors'].items():
rgba = [float(c) / 255.0 for c in rgb] + [1.0]
AVAILABLE_COLOURS.append((name, rgba))
AVAILABLE_MATERIALS = [(v, k)
for k, v in properties['materials'].items()]
AVAILABLE_OBJECTS = [(v, k)
for k, v in properties['shapes'].items()]
AVAILABLE_SIZES = list(properties['sizes'].items())
NUM_AVAILABLE_OBJECTS = len(AVAILABLE_OBJECTS)
NUM_AVAILABLE_MATERIALS = len(AVAILABLE_MATERIALS)
NUM_AVAILABLE_SIZES = len(AVAILABLE_SIZES)
NUM_AVAILABLE_COLOURS = len(AVAILABLE_COLOURS)
# categorical probabilities
obj_probs = torch.ones(
NUM_AVAILABLE_OBJECTS) / NUM_AVAILABLE_OBJECTS
material_probs = torch.ones(
NUM_AVAILABLE_MATERIALS) / NUM_AVAILABLE_MATERIALS
colour_probs = torch.ones(
NUM_AVAILABLE_COLOURS) / NUM_AVAILABLE_COLOURS
size_probs = torch.ones(NUM_AVAILABLE_SIZES) / NUM_AVAILABLE_SIZES
except:
print("Unable to open properties file (properties_json argument)")
exit()
# OOD extrapolation: add object (out of training set)
if args.out_of_distribution == 1:
AVAILABLE_OBJECTS.append(('Cone', 'cone'))
NUM_AVAILABLE_OBJECTS += 1
obj_probs = torch.ones(NUM_AVAILABLE_OBJECTS) / NUM_AVAILABLE_OBJECTS
elif args.out_of_distribution == 2:
AVAILABLE_OBJECTS.append(('Corgi', 'corgi'))
NUM_AVAILABLE_OBJECTS += 1
obj_probs = torch.ones(NUM_AVAILABLE_OBJECTS) / NUM_AVAILABLE_OBJECTS
# adversarial or OOD extrapolation: remove object
if args.remove_object_type != None:
NEW_AVAILABLE_OBJECTS = []
for i in range(len(AVAILABLE_OBJECTS)):
_, object_name = AVAILABLE_OBJECTS[i]
if object_name != args.remove_object_type:
NEW_AVAILABLE_OBJECTS.append(AVAILABLE_OBJECTS[i])
AVAILABLE_OBJECTS = NEW_AVAILABLE_OBJECTS
NUM_AVAILABLE_OBJECTS = len(AVAILABLE_OBJECTS)
obj_probs = torch.ones(NUM_AVAILABLE_OBJECTS) / NUM_AVAILABLE_OBJECTS
if args.save_dir != None:
save_directory = args.save_dir
if args.baseline_model is not None:
print('Loading baseline model from ', args.baseline_model)
model, _ = utils.load_baseline(args.baseline_model)
if args.vocab_json is not None:
new_vocab = utils.load_vocab(args.vocab_json)
model.rnn.expand_vocab(new_vocab['question_token_to_idx'])
elif args.program_generator is not None and args.execution_engine is not None:
print('Loading program generator from ', args.program_generator)
program_generator, _ = utils.load_program_generator(
args.program_generator)
print('Loading execution engine from ', args.execution_engine)
execution_engine, _ = utils.load_execution_engine(
args.execution_engine, verbose=False)
if args.vocab_json is not None:
new_vocab = utils.load_vocab(args.vocab_json)
program_generator.expand_encoder_vocab(
new_vocab['question_token_to_idx'])
model = (program_generator, execution_engine)
else:
print(
'Must give either --baseline_model or --program_generator and --execution_engine'
)
return
print("Calling inference!")
random_latent = generate_random_latent(num_objects=args.num_objects)
print(random_latent)
if args.prob_test == 0:
print("Running Metropolis Hastings (one constraint)")
metropolis_hastings(initial_proposal=random_latent,
num_iters=int(args.num_iters),
std=0.05,
args=args,
model=model,
target_class=args.class_a,
num_objects=args.num_objects,
output_csv=args.output_csv,
test_name=args.test_name)
if args.prob_test == 1:
print("Running Metropolis Hastings (two constraints)")
target_classes = [args.class_a, args.class_b]
metropolis_hastings_two_classes(initial_proposal=random_latent,
num_iters=int(args.num_iters),
std=0.05,
args=args,
model=model,
target_classes=target_classes,
num_objects=args.num_objects)
if args.prob_test == 2:
print("Running Rejection Sampling (one constraint)")
rejection_sampling(initial_proposal=random_latent,
num_iters=int(args.num_iters),
args=args,
model=model,
target_class=args.class_a,
num_objects=args.num_objects,
output_csv=args.output_csv,
test_name=args.test_name)
