def test_context_memory(config_filename, search_engine, src_sentence):
config_training = train_config.load_config_train(config_filename)
(encdec, eos_idx, src_indexer, tgt_indexer
), model_infos = train.create_encdec_and_indexers_from_config_dict(
config_training, load_config_model="yes", return_model_infos=True)
ctxt_mem = create_context_memory(encdec, (src_indexer, tgt_indexer),
search_engine, src_sentence)
def create_and_load_encdec_from_files(config_training_fn, trained_model):
log.info("loading model config from %s" % config_training_fn)
config_training = train_config.load_config_train(config_training_fn)
encdec, eos_idx, src_indexer, tgt_indexer = train.create_encdec_and_indexers_from_config_dict(config_training)
log.info("loading model from %s" % trained_model)
serializers.load_npz(trained_model, encdec)
return encdec, eos_idx, src_indexer, tgt_indexer
def create_encdec(config_eval):
encdec_list = []
eos_idx, src_indexer, tgt_indexer = None, None, None
model_infos_list = []
if config_eval.training_config is not None:
assert config_eval.trained_model is not None
encdec, eos_idx, src_indexer, tgt_indexer = create_and_load_encdec_from_files(
config_eval.training_config, config_eval.trained_model)
model_infos_list.append(
create_filename_infos(config_eval.trained_model))
encdec_list.append(encdec)
if 'load_model_config' in config_eval.process and config_eval.process.load_model_config is not None:
for config_filename_and_others in config_eval.process.load_model_config:
other_models_for_averaging = None
if "," in config_filename_and_others:
config_filename_and_others_splitted = config_filename_and_others.split(
",")
config_filename = config_filename_and_others_splitted[0]
other_models_for_averaging = config_filename_and_others_splitted[
1:]
else:
config_filename = config_filename_and_others
log.info("loading model and parameters from config %s" %
config_filename)
config_training = train_config.load_config_train(config_filename)
(
encdec, this_eos_idx, this_src_indexer, this_tgt_indexer
), model_infos = train.create_encdec_and_indexers_from_config_dict(
config_training,
load_config_model="yes",
return_model_infos=True,
additional_models_parameters_for_averaging=
other_models_for_averaging)
model_infos_list.append(model_infos)
if eos_idx is None:
assert len(encdec_list) == 0
assert src_indexer is None
assert tgt_indexer is None
eos_idx, src_indexer, tgt_indexer = this_eos_idx, this_src_indexer, this_tgt_indexer
else:
check_if_vocabulary_info_compatible(this_eos_idx,
this_src_indexer,
this_tgt_indexer, eos_idx,
src_indexer, tgt_indexer)
encdec_list.append(encdec)
assert len(encdec_list) > 0
if 'additional_training_config' in config_eval.process and config_eval.process.additional_training_config is not None:
assert len(config_eval.process.additional_training_config) == len(
config_eval.process.additional_trained_model)
for (config_training_fn, trained_model_fn) in six.moves.zip(
config_eval.process.additional_training_config,
config_eval.process.additional_trained_model):
this_encdec, this_eos_idx, this_src_indexer, this_tgt_indexer = create_and_load_encdec_from_files(
config_training_fn, trained_model_fn)
check_if_vocabulary_info_compatible(this_eos_idx, this_src_indexer,
this_tgt_indexer, eos_idx,
src_indexer, tgt_indexer)
model_infos_list.append(create_filename_infos(trained_model_fn))
encdec_list.append(this_encdec)
if config_eval.process.use_chainerx:
if 'gpu' in config_eval.process and config_eval.process.gpu is not None:
encdec_list = [
encdec.to_device("cuda:%i" % config_eval.process.gpu)
for encdec in encdec_list
]
else:
encdec_list = [
encdec.to_device("native:0") for encdec in encdec_list
]
else:
if 'gpu' in config_eval.process and config_eval.process.gpu is not None:
encdec_list = [
encdec.to_gpu(config_eval.process.gpu)
for encdec in encdec_list
]
if 'reverse_training_config' in config_eval.process and config_eval.process.reverse_training_config is not None:
reverse_encdec, reverse_eos_idx, reverse_src_indexer, reverse_tgt_indexer = create_and_load_encdec_from_files(
config_eval.process.reverse_training_config,
config_eval.process.reverse_trained_model)
if eos_idx != reverse_eos_idx:
raise Exception("incompatible models")
if len(src_indexer) != len(reverse_src_indexer):
raise Exception("incompatible models")
if len(tgt_indexer) != len(reverse_tgt_indexer):
raise Exception("incompatible models")
if config_eval.process.gpu is not None:
reverse_encdec = reverse_encdec.to_gpu(config_eval.process.gpu)
else:
reverse_encdec = None
return encdec_list, eos_idx, src_indexer, tgt_indexer, reverse_encdec, model_infos_list
def create_encdec(config_eval):
encdec_list = []
eos_idx, src_indexer, tgt_indexer = None, None, None
model_infos_list = []
if config_eval.training_config is not None:
assert config_eval.trained_model is not None
encdec, eos_idx, src_indexer, tgt_indexer = create_and_load_encdec_from_files(
config_eval.training_config, config_eval.trained_model)
model_infos_list.append(create_filename_infos(config_eval.trained_model))
encdec_list.append(encdec)
if 'load_model_config' in config_eval.process and config_eval.process.load_model_config is not None:
for config_filename in config_eval.process.load_model_config:
log.info(
"loading model and parameters from config %s" %
config_filename)
config_training = train_config.load_config_train(config_filename)
(encdec, this_eos_idx, this_src_indexer, this_tgt_indexer), model_infos = train.create_encdec_and_indexers_from_config_dict(config_training,
load_config_model="yes",
return_model_infos=True)
model_infos_list.append(model_infos)
if eos_idx is None:
assert len(encdec_list) == 0
assert src_indexer is None
assert tgt_indexer is None
eos_idx, src_indexer, tgt_indexer = this_eos_idx, this_src_indexer, this_tgt_indexer
else:
check_if_vocabulary_info_compatible(this_eos_idx, this_src_indexer, this_tgt_indexer, eos_idx, src_indexer, tgt_indexer)
encdec_list.append(encdec)
assert len(encdec_list) > 0
if 'additional_training_config' in config_eval.process and config_eval.process.additional_training_config is not None:
assert len(config_eval.process.additional_training_config) == len(config_eval.process.additional_trained_model)
for (config_training_fn, trained_model_fn) in zip(config_eval.process.additional_training_config,
config_eval.process.additional_trained_model):
this_encdec, this_eos_idx, this_src_indexer, this_tgt_indexer = create_and_load_encdec_from_files(
config_training_fn, trained_model_fn)
check_if_vocabulary_info_compatible(this_eos_idx, this_src_indexer, this_tgt_indexer, eos_idx, src_indexer, tgt_indexer)
model_infos_list.append(create_filename_infos(trained_model_fn))
# if args.gpu is not None:
# this_encdec = this_encdec.to_gpu(args.gpu)
encdec_list.append(this_encdec)
if 'gpu' in config_eval.process and config_eval.process.gpu is not None:
encdec_list = [encdec.to_gpu(config_eval.process.gpu) for encdec in encdec_list]
if 'reverse_training_config' in config_eval.process and config_eval.process.reverse_training_config is not None:
reverse_encdec, reverse_eos_idx, reverse_src_indexer, reverse_tgt_indexer = create_and_load_encdec_from_files(
config_eval.process.reverse_training_config, config_eval.process.reverse_trained_model)
if eos_idx != reverse_eos_idx:
raise Exception("incompatible models")
if len(src_indexer) != len(reverse_src_indexer):
raise Exception("incompatible models")
if len(tgt_indexer) != len(reverse_tgt_indexer):
raise Exception("incompatible models")
if config_eval.process.gpu is not None:
reverse_encdec = reverse_encdec.to_gpu(config_eval.process.gpu)
else:
reverse_encdec = None
return encdec_list, eos_idx, src_indexer, tgt_indexer, reverse_encdec, model_infos_list