def load_execution_engine(path, verbose=True):
checkpoint = load_cpu(path)
if checkpoint['args'].get('symbolic_ee'):
vocab = load_vocab(checkpoint['args']['vocab_json'])
ee = ClevrExecutor(vocab)
return ee, {}
model_type = checkpoint['args']['model_type']
kwargs = checkpoint['execution_engine_kwargs']
state = checkpoint['execution_engine_state']
kwargs['verbose'] = verbose
if model_type == 'FiLM':
model = FiLMedNet(**kwargs)
elif model_type in ['PG+EE', 'EE', 'Control-EE']:
kwargs.pop('sharing_patterns', None)
kwargs.setdefault('module_pool', 'mean')
kwargs.setdefault('module_use_gammas', 'linear')
model = ModuleNet(**kwargs)
elif model_type == 'MAC':
kwargs.setdefault('write_unit', 'original')
kwargs.setdefault('read_connect', 'last')
kwargs.setdefault('read_unit', 'original')
kwargs.setdefault('noisy_controls', False)
kwargs.pop('sharing_params_patterns', None)
model = MAC(**kwargs)
elif model_type == 'RelNet':
model = RelationNet(**kwargs)
elif model_type == 'SHNMN':
model = SHNMN(**kwargs)
elif model_type == 'SimpleNMN':
model = SimpleModuleNet(**kwargs)
else:
raise ValueError()
cur_state = model.state_dict()
model.load_state_dict(state)
return model, kwargs
def load_execution_engine(path, verbose=True):
checkpoint = load_cpu(path)
model_type = checkpoint['args']['model_type']
kwargs = checkpoint['execution_engine_kwargs']
state = checkpoint['execution_engine_state']
kwargs['verbose'] = verbose
if model_type == 'FiLM':
kwargs = get_updated_args(kwargs, FiLMedNet)
model = FiLMedNet(**kwargs)
elif model_type == 'EE':
model = ModuleNet(**kwargs)
elif model_type == 'MAC':
kwargs.setdefault('write_unit', 'original')
kwargs.setdefault('read_connect', 'last')
kwargs.setdefault('noisy_controls', False)
kwargs.pop('sharing_params_patterns', None)
model = MAC(**kwargs)
elif model_type == 'RelNet':
model = RelationNet(**kwargs)
elif model_type == 'SHNMN':
model = SHNMN(**kwargs)
else:
raise ValueError()
cur_state = model.state_dict()
model.load_state_dict(state)
return model, kwargs
def get_execution_engine(args):
vocab = utils.load_vocab(args.vocab_json)
if args.execution_engine_start_from is not None:
ee, kwargs = utils.load_execution_engine(
args.execution_engine_start_from, model_type=args.model_type)
else:
kwargs = {
'vocab': vocab,
'feature_dim': parse_int_list(args.feature_dim),
'stem_batchnorm': args.module_stem_batchnorm == 1,
'stem_num_layers': args.module_stem_num_layers,
'module_dim': args.module_dim,
'module_residual': args.module_residual == 1,
'module_batchnorm': args.module_batchnorm == 1,
'classifier_proj_dim': args.classifier_proj_dim,
'classifier_downsample': args.classifier_downsample,
'classifier_fc_layers': parse_int_list(args.classifier_fc_dims),
'classifier_batchnorm': args.classifier_batchnorm == 1,
'classifier_dropout': args.classifier_dropout,
'encoder_vocab_size': len(vocab['question_token_to_idx']),
'decoder_vocab_size': len(vocab['program_token_to_idx']),
'wordvec_dim': args.rnn_wordvec_dim,
'hidden_dim': args.rnn_hidden_dim,
'rnn_num_layers': args.rnn_num_layers,
'rnn_dropout': args.rnn_dropout, # 0e-2
}
if args.model_type == 'FiLM':
kwargs['num_modules'] = args.num_modules
kwargs['stem_kernel_size'] = args.module_stem_kernel_size
kwargs['stem_stride'] = args.module_stem_stride
kwargs['stem_padding'] = args.module_stem_padding
kwargs['module_num_layers'] = args.module_num_layers
kwargs['module_batchnorm_affine'] = args.module_batchnorm_affine == 1
kwargs['module_dropout'] = args.module_dropout
kwargs['module_input_proj'] = args.module_input_proj
kwargs['module_kernel_size'] = args.module_kernel_size
kwargs['use_gamma'] = args.use_gamma == 1
kwargs['use_beta'] = args.use_beta == 1
kwargs['use_coords'] = args.use_coords
kwargs['debug_every'] = args.debug_every
kwargs['print_verbose_every'] = args.print_verbose_every
kwargs['condition_method'] = args.condition_method
kwargs['condition_pattern'] = parse_int_list(args.condition_pattern)
kwargs['parameter_efficient'] = args.program_generator_parameter_efficient == 1
kwargs['output_batchnorm'] = args.rnn_output_batchnorm == 1
kwargs['bidirectional'] = args.bidirectional == 1
kwargs['rnn_time_step'] = args.rnn_time_step
kwargs['encoder_type'] = args.encoder_type
kwargs['decoder_type'] = args.decoder_type
kwargs['gamma_option'] = args.gamma_option
kwargs['gamma_baseline'] = args.gamma_baseline # 1
ee = FiLMedNet(**kwargs)
else:
ee = ModuleNet(**kwargs)
ee.cuda()
ee.train()
return ee, kwargs
def get_execution_engine(args):
vocab = utils.load_vocab(args.vocab_json)
if args.execution_engine_start_from is not None:
ee, kwargs = utils.load_execution_engine(
args.execution_engine_start_from, model_type=args.model_type)
else:
kwargs = {
'vocab': vocab,
'feature_dim': parse_int_list(args.feature_dim),
'stem_batchnorm': args.module_stem_batchnorm == 1,
'stem_num_layers': args.module_stem_num_layers,
'module_dim': args.module_dim,
'module_residual': args.module_residual == 1,
'module_batchnorm': args.module_batchnorm == 1,
'classifier_proj_dim': args.classifier_proj_dim,
'classifier_downsample': args.classifier_downsample,
'classifier_fc_layers': parse_int_list(args.classifier_fc_dims),
'classifier_batchnorm': args.classifier_batchnorm == 1,
'classifier_dropout': args.classifier_dropout,
}
if args.model_type.startswith('FiLM'):
kwargs['num_modules'] = args.num_modules
kwargs['stem_use_resnet'] = (args.model_type == 'FiLM+ResNet1' or args.model_type == 'FiLM+ResNet0')
kwargs['stem_resnet_fixed'] = args.model_type == 'FiLM+ResNet0'
kwargs['stem_kernel_size'] = args.module_stem_kernel_size
kwargs['stem_stride2_freq'] = args.module_stem_stride2_freq
kwargs['stem_padding'] = args.module_stem_padding
kwargs['module_num_layers'] = args.module_num_layers
kwargs['module_batchnorm_affine'] = args.module_batchnorm_affine == 1
kwargs['module_dropout'] = args.module_dropout
kwargs['module_input_proj'] = args.module_input_proj
kwargs['module_kernel_size'] = args.module_kernel_size
kwargs['use_gamma'] = args.use_gamma == 1
kwargs['use_beta'] = args.use_beta == 1
kwargs['use_coords'] = args.use_coords
kwargs['debug_every'] = args.debug_every
kwargs['print_verbose_every'] = args.print_verbose_every
kwargs['condition_method'] = args.condition_method
kwargs['condition_pattern'] = parse_int_list(args.condition_pattern)
ee = FiLMedNet(**kwargs)
else:
ee = ModuleNet(**kwargs)
if torch.cuda.is_available():
ee.cuda()
else:
ee.cpu()
ee.train()
return ee, kwargs
def get_execution_engine(args):
vocab = utils.load_vocab(args.vocab_json)
if args.execution_engine_start_from is not None:
ee, kwargs = utils.load_execution_engine(
args.execution_engine_start_from, model_type=args.model_type)
else:
kwargs = {
'vocab': vocab,
'feature_dim': parse_int_list(args.feature_dim),
'stem_batchnorm': args.module_stem_batchnorm == 1,
'stem_num_layers': args.module_stem_num_layers,
'module_dim': args.module_dim,
'module_residual': args.module_residual == 1,
'module_batchnorm': args.module_batchnorm == 1,
'classifier_proj_dim': args.classifier_proj_dim,
'classifier_downsample': args.classifier_downsample,
'classifier_fc_layers': parse_int_list(args.classifier_fc_dims),
'classifier_batchnorm': args.classifier_batchnorm == 1,
'classifier_dropout': args.classifier_dropout,
}
if args.model_type == 'FiLM':
kwargs['num_modules'] = args.num_modules
kwargs['stem_kernel_size'] = args.module_stem_kernel_size
kwargs['stem_stride'] = args.module_stem_stride
kwargs['stem_padding'] = args.module_stem_padding
kwargs['module_num_layers'] = args.module_num_layers
kwargs[
'module_batchnorm_affine'] = args.module_batchnorm_affine == 1
kwargs['module_dropout'] = args.module_dropout
kwargs['module_input_proj'] = args.module_input_proj
kwargs['module_kernel_size'] = args.module_kernel_size
kwargs['use_gamma'] = args.use_gamma == 1
kwargs['use_beta'] = args.use_beta == 1
kwargs['use_coords'] = args.use_coords
kwargs['debug_every'] = args.debug_every
kwargs['print_verbose_every'] = args.print_verbose_every
kwargs['condition_method'] = args.condition_method
kwargs['with_cbn'] = args.with_cbn
kwargs['final_resblock_with_cbn'] = args.final_resblock_with_cbn
kwargs['condition_pattern'] = parse_int_list(
args.condition_pattern)
ee = FiLMedNet(**kwargs)
else:
ee = ModuleNet(**kwargs)
# if cuda.device_count() > 1:
# ee = nn.DataParallel(ee)
ee.cuda()
ee.train()
return ee, kwargs
def load_execution_engine(path, verbose=True, model_type='PG+EE'):
checkpoint = load_cpu(path)
kwargs = checkpoint['execution_engine_kwargs']
state = checkpoint['execution_engine_state']
kwargs['verbose'] = verbose
if model_type == 'FiLM':
print('Loading FiLMedNet from ' + path)
kwargs = get_updated_args(kwargs, FiLMedNet)
model = FiLMedNet(**kwargs)
else:
print('Loading EE from ' + path)
model = ModuleNet(**kwargs)
cur_state = model.state_dict()
model.load_state_dict(state)
return model, kwargs
def load_execution_engine(path, verbose=True, model_type="PG+EE"):
checkpoint = load_cpu(path)
kwargs = checkpoint["execution_engine_kwargs"]
state = checkpoint["execution_engine_state"]
kwargs["verbose"] = verbose
if model_type == "FiLM":
print("Loading FiLMedNet from " + path)
kwargs = get_updated_args(kwargs, FiLMedNet)
model = FiLMedNet(**kwargs)
else:
print("Loading EE from " + path)
model = ModuleNet(**kwargs)
cur_state = model.state_dict()
model.load_state_dict(state)
return model, kwargs
def load_execution_engine(path, verbose=True, model_type='PG+EE'):
checkpoint = load_cpu(path)
kwargs = checkpoint['execution_engine_kwargs']
state = checkpoint['execution_engine_state']
kwargs['verbose'] = verbose
if model_type == 'FiLM':
print('Loading FiLMedNet from ' + path)
kwargs = get_updated_args(kwargs, FiLMedNet)
model = FiLMedNet(**kwargs)
else:
print('Loading EE from ' + path)
model = ModuleNet(**kwargs)
cur_state = model.state_dict()
# cur_params = dict(model.named_parameters())
# print cur_params.keys()
state_stemed = {}
for k, v in state.iteritems():
k_new = '.'.join(k.split('.')[1:])
state_stemed[k_new] = v
model.load_state_dict(state_stemed)
return model, kwargs
film_gen = film_gen.cuda()
filmed_net = FiLMedNet(
vocab,
feature_dim=(1024, 14, 14),
stem_num_layers=args.module_stem_num_layers,
stem_batchnorm=args.module_stem_batchnorm,
stem_kernel_size=args.module_stem_kernel_size,
stem_stride=1,
stem_padding=None,
num_modules=args.num_modules,
module_num_layers=args.module_num_layers,
module_dim=args.module_dim,
module_residual=args.module_residual,
module_batchnorm=args.module_batchnorm,
module_batchnorm_affine=args.module_batchnorm_affine,
module_dropout=args.module_dropout,
module_input_proj=args.module_input_proj,
module_kernel_size=args.module_kernel_size,
classifier_proj_dim=args.classifier_proj_dim,
classifier_downsample='maxpoolfull',
classifier_fc_layers=(1024, ),
classifier_batchnorm=args.classifier_batchnorm,
classifier_dropout=0,
condition_method='bn-film',
condition_pattern=[],
use_gamma=True,
use_beta=True,
use_coords=args.use_coords,
late_fusion_question=(args.classifier_late_fusion == 1),
rnn_hidden_dim=args.rnn_hidden_dim)
filmed_net = filmed_net.cuda()
