def create_encdec_and_indexers_from_config_dict(config_dict, src_indexer=None, tgt_indexer=None, load_config_model="no",
return_model_infos=False):
assert load_config_model in "yes no if_exists".split()
if src_indexer is None or tgt_indexer is None:
voc_fn = config_dict.data["voc"]
log.info("loading voc from %s" % voc_fn)
# src_voc, tgt_voc = json.load(open(voc_fn))
bi_idx = processors.load_pp_pair_from_file(voc_fn)
if src_indexer is None:
src_indexer = bi_idx.src_processor()
if tgt_indexer is None:
tgt_indexer = bi_idx.tgt_processor()
# tgt_voc = None
# src_voc = None
encdec = create_encdec_from_config_dict(config_dict["model"], src_indexer, tgt_indexer)
eos_idx = len(tgt_indexer)
model_infos = None
if load_config_model != "no":
if "model_parameters" not in config_dict:
if load_config_model == "yes":
log.error("cannot find model parameters in config file")
raise ValueError(
"Config file do not contain model_parameters section")
else:
if config_dict.model_parameters.type == "model":
model_filename = config_dict.model_parameters.filename
log.info(
"loading model parameters from file specified by config file:%s" %
model_filename)
serializers.load_npz(model_filename, encdec)
if return_model_infos:
model_infos = create_filename_infos(model_filename)
else:
if load_config_model == "yes":
log.error(
"model parameters in config file is of type snapshot, not model")
raise ValueError("Config file model is not of type model")
result = encdec, eos_idx, src_indexer, tgt_indexer
if return_model_infos:
return result, model_infos
else:
return result
def create_encdec(config_eval):
encdec_list = []
eos_idx, src_indexer, tgt_indexer = None, None, None
model_infos_list = []
if config_eval.training_config is not None:
assert config_eval.trained_model is not None
encdec, eos_idx, src_indexer, tgt_indexer = create_and_load_encdec_from_files(
config_eval.training_config, config_eval.trained_model)
model_infos_list.append(
create_filename_infos(config_eval.trained_model))
encdec_list.append(encdec)
if 'load_model_config' in config_eval.process and config_eval.process.load_model_config is not None:
for config_filename_and_others in config_eval.process.load_model_config:
other_models_for_averaging = None
if "," in config_filename_and_others:
config_filename_and_others_splitted = config_filename_and_others.split(
",")
config_filename = config_filename_and_others_splitted[0]
other_models_for_averaging = config_filename_and_others_splitted[
1:]
else:
config_filename = config_filename_and_others
log.info("loading model and parameters from config %s" %
config_filename)
config_training = train_config.load_config_train(config_filename)
(
encdec, this_eos_idx, this_src_indexer, this_tgt_indexer
), model_infos = train.create_encdec_and_indexers_from_config_dict(
config_training,
load_config_model="yes",
return_model_infos=True,
additional_models_parameters_for_averaging=
other_models_for_averaging)
model_infos_list.append(model_infos)
if eos_idx is None:
assert len(encdec_list) == 0
assert src_indexer is None
assert tgt_indexer is None
eos_idx, src_indexer, tgt_indexer = this_eos_idx, this_src_indexer, this_tgt_indexer
else:
check_if_vocabulary_info_compatible(this_eos_idx,
this_src_indexer,
this_tgt_indexer, eos_idx,
src_indexer, tgt_indexer)
encdec_list.append(encdec)
assert len(encdec_list) > 0
if 'additional_training_config' in config_eval.process and config_eval.process.additional_training_config is not None:
assert len(config_eval.process.additional_training_config) == len(
config_eval.process.additional_trained_model)
for (config_training_fn, trained_model_fn) in six.moves.zip(
config_eval.process.additional_training_config,
config_eval.process.additional_trained_model):
this_encdec, this_eos_idx, this_src_indexer, this_tgt_indexer = create_and_load_encdec_from_files(
config_training_fn, trained_model_fn)
check_if_vocabulary_info_compatible(this_eos_idx, this_src_indexer,
this_tgt_indexer, eos_idx,
src_indexer, tgt_indexer)
model_infos_list.append(create_filename_infos(trained_model_fn))
encdec_list.append(this_encdec)
if config_eval.process.use_chainerx:
if 'gpu' in config_eval.process and config_eval.process.gpu is not None:
encdec_list = [
encdec.to_device("cuda:%i" % config_eval.process.gpu)
for encdec in encdec_list
]
else:
encdec_list = [
encdec.to_device("native:0") for encdec in encdec_list
]
else:
if 'gpu' in config_eval.process and config_eval.process.gpu is not None:
encdec_list = [
encdec.to_gpu(config_eval.process.gpu)
for encdec in encdec_list
]
if 'reverse_training_config' in config_eval.process and config_eval.process.reverse_training_config is not None:
reverse_encdec, reverse_eos_idx, reverse_src_indexer, reverse_tgt_indexer = create_and_load_encdec_from_files(
config_eval.process.reverse_training_config,
config_eval.process.reverse_trained_model)
if eos_idx != reverse_eos_idx:
raise Exception("incompatible models")
if len(src_indexer) != len(reverse_src_indexer):
raise Exception("incompatible models")
if len(tgt_indexer) != len(reverse_tgt_indexer):
raise Exception("incompatible models")
if config_eval.process.gpu is not None:
reverse_encdec = reverse_encdec.to_gpu(config_eval.process.gpu)
else:
reverse_encdec = None
return encdec_list, eos_idx, src_indexer, tgt_indexer, reverse_encdec, model_infos_list
def create_encdec_and_indexers_from_config_dict(
config_dict,
src_indexer=None,
tgt_indexer=None,
load_config_model="no",
return_model_infos=False,
additional_models_parameters_for_averaging=None):
assert load_config_model in "yes no if_exists".split()
if src_indexer is None or tgt_indexer is None:
voc_fn = config_dict.data["voc"]
log.info("loading voc from %s" % voc_fn)
# src_voc, tgt_voc = json.load(open(voc_fn))
bi_idx = processors.load_pp_pair_from_file(voc_fn)
if src_indexer is None:
src_indexer = bi_idx.src_processor()
if tgt_indexer is None:
tgt_indexer = bi_idx.tgt_processor()
# tgt_voc = None
# src_voc = None
encdec = create_encdec_from_config_dict(config_dict["model"], src_indexer,
tgt_indexer)
eos_idx = len(tgt_indexer)
model_infos = None
if load_config_model != "no":
if "model_parameters" not in config_dict:
assert additional_models_parameters_for_averaging is None
if load_config_model == "yes":
log.error("cannot find model parameters in config file")
raise ValueError(
"Config file do not contain model_parameters section")
else:
model_filename = config_dict.model_parameters.filename
if additional_models_parameters_for_averaging is not None:
load_model_flexible([model_filename] +
additional_models_parameters_for_averaging,
encdec)
else:
load_model_flexible(model_filename, encdec)
# if config_dict.model_parameters.type == "model":
# log.info(
# "loading model parameters from file specified by config file:%s" %
# model_filename)
# serializers.load_npz(model_filename, encdec)
# if return_model_infos:
# model_infos = create_filename_infos(model_filename)
# else:
# log.info("loading model parameters from snapshot file specified by config file:%s" %model_filename)
# with np.load(model_filename) as fs:
# dics = serializers.NpzDeserializer(fs, path="updater/model:main/")
# dics.load(encdec)
if return_model_infos:
model_infos = create_filename_infos(model_filename)
else:
assert additional_models_parameters_for_averaging is None
result = encdec, eos_idx, src_indexer, tgt_indexer
if return_model_infos:
return result, model_infos
else:
return result
def create_encdec(config_eval):
encdec_list = []
eos_idx, src_indexer, tgt_indexer = None, None, None
model_infos_list = []
if config_eval.training_config is not None:
assert config_eval.trained_model is not None
encdec, eos_idx, src_indexer, tgt_indexer = create_and_load_encdec_from_files(
config_eval.training_config, config_eval.trained_model)
model_infos_list.append(create_filename_infos(config_eval.trained_model))
encdec_list.append(encdec)
if 'load_model_config' in config_eval.process and config_eval.process.load_model_config is not None:
for config_filename in config_eval.process.load_model_config:
log.info(
"loading model and parameters from config %s" %
config_filename)
config_training = train_config.load_config_train(config_filename)
(encdec, this_eos_idx, this_src_indexer, this_tgt_indexer), model_infos = train.create_encdec_and_indexers_from_config_dict(config_training,
load_config_model="yes",
return_model_infos=True)
model_infos_list.append(model_infos)
if eos_idx is None:
assert len(encdec_list) == 0
assert src_indexer is None
assert tgt_indexer is None
eos_idx, src_indexer, tgt_indexer = this_eos_idx, this_src_indexer, this_tgt_indexer
else:
check_if_vocabulary_info_compatible(this_eos_idx, this_src_indexer, this_tgt_indexer, eos_idx, src_indexer, tgt_indexer)
encdec_list.append(encdec)
assert len(encdec_list) > 0
if 'additional_training_config' in config_eval.process and config_eval.process.additional_training_config is not None:
assert len(config_eval.process.additional_training_config) == len(config_eval.process.additional_trained_model)
for (config_training_fn, trained_model_fn) in zip(config_eval.process.additional_training_config,
config_eval.process.additional_trained_model):
this_encdec, this_eos_idx, this_src_indexer, this_tgt_indexer = create_and_load_encdec_from_files(
config_training_fn, trained_model_fn)
check_if_vocabulary_info_compatible(this_eos_idx, this_src_indexer, this_tgt_indexer, eos_idx, src_indexer, tgt_indexer)
model_infos_list.append(create_filename_infos(trained_model_fn))
# if args.gpu is not None:
# this_encdec = this_encdec.to_gpu(args.gpu)
encdec_list.append(this_encdec)
if 'gpu' in config_eval.process and config_eval.process.gpu is not None:
encdec_list = [encdec.to_gpu(config_eval.process.gpu) for encdec in encdec_list]
if 'reverse_training_config' in config_eval.process and config_eval.process.reverse_training_config is not None:
reverse_encdec, reverse_eos_idx, reverse_src_indexer, reverse_tgt_indexer = create_and_load_encdec_from_files(
config_eval.process.reverse_training_config, config_eval.process.reverse_trained_model)
if eos_idx != reverse_eos_idx:
raise Exception("incompatible models")
if len(src_indexer) != len(reverse_src_indexer):
raise Exception("incompatible models")
if len(tgt_indexer) != len(reverse_tgt_indexer):
raise Exception("incompatible models")
if config_eval.process.gpu is not None:
reverse_encdec = reverse_encdec.to_gpu(config_eval.process.gpu)
else:
reverse_encdec = None
return encdec_list, eos_idx, src_indexer, tgt_indexer, reverse_encdec, model_infos_list