def train(self,
config,
src_file,
tgt_file,
src_vocab_info,
tgt_vocab_info,
align_file=None,
model_path=None,
gpuid=0):
if src_vocab_info['changed'] or tgt_vocab_info['changed']:
model_path = checkpoint.update_vocab(
model_path,
os.path.join(self._output_dir, 'new_vocab_checkpoint'),
src_vocab_info['model'],
tgt_vocab_info['model'],
new_src_vocab=src_vocab_info['current']
if src_vocab_info['changed'] else None,
new_tgt_vocab=tgt_vocab_info['current']
if tgt_vocab_info['changed'] else None,
mode='replace',
session_config=tf.ConfigProto(device_count={'GPU': 0}))
model_dir, model = self._load_model(
model_type=config['options'].get('model_type'),
model_file=config['options'].get('model'),
model_path=model_path)
run_config = copy.deepcopy(config['options'].get('config', {}))
run_config['model_dir'] = model_dir
if 'data' not in run_config:
run_config['data'] = {}
if 'train' not in run_config:
run_config['train'] = {}
run_config['data']['source_words_vocabulary'] = src_vocab_info[
'current']
run_config['data']['target_words_vocabulary'] = tgt_vocab_info[
'current']
run_config['data']['train_features_file'] = src_file
run_config['data']['train_labels_file'] = tgt_file
if align_file is not None and os.path.exists(align_file):
run_config['data']['train_alignments'] = align_file
if "params" not in run_config:
run_config["params"] = {}
if "guided_alignment_type" not in run_config["params"]:
run_config["params"]["guided_alignment_type"] = "ce"
if 'train_steps' not in run_config['train']:
run_config['train']['single_pass'] = True
run_config['train']['train_steps'] = None
if 'sample_buffer_size' not in run_config['train']:
run_config['train']['sample_buffer_size'] = -1
if 'average_last_checkpoints' not in run_config['train']:
run_config['train']['average_last_checkpoints'] = 0
runner = onmt.Runner(model,
run_config,
num_devices=utils.count_devices(gpuid),
auto_config=config['options'].get(
'auto_config', False))
output_dir = runner.train()
if output_dir != model_dir:
shutil.copy(os.path.join(model_dir, "model_description.py"),
output_dir)
return self._list_model_files(output_dir)
def train(
self,
config,
src_file,
tgt_file,
src_vocab_info,
tgt_vocab_info,
align_file=None,
example_weights_file=None,
model_path=None,
gpuid=0,
):
if model_path is None or tf.train.latest_checkpoint(
model_path) is None:
prev_src_vocab = None
prev_tgt_vocab = None
else:
prev_src_vocab = src_vocab_info.previous
prev_tgt_vocab = tgt_vocab_info.previous
runner = self._build_runner(
config,
src_vocab=prev_src_vocab or src_vocab_info.current,
tgt_vocab=prev_tgt_vocab or tgt_vocab_info.current,
src_file=src_file,
tgt_file=tgt_file,
align_file=align_file,
example_weights_file=example_weights_file,
model_path=model_path,
)
if prev_src_vocab or prev_tgt_vocab:
previous_model_dir = runner.model_dir
# Update the model vocabulary on CPU to avoid initializing the GPU context
# and allow batch size autotuning to run properly afterwards.
with tf.device("cpu"):
runner.update_vocab(
os.path.join(self._output_dir, "new_vocab_checkpoint"),
src_vocab=src_vocab_info.current
if prev_src_vocab else None,
tgt_vocab=tgt_vocab_info.current
if prev_tgt_vocab else None,
)
shutil.rmtree(previous_model_dir)
output_dir, summary = runner.train(
num_devices=utils.count_devices(gpuid),
return_summary=True,
fallback_to_cpu=False,
)
return _list_checkpoint_files(output_dir), summary
def train(self,
config,
src_file,
tgt_file,
src_vocab_info,
tgt_vocab_info,
model_path=None,
gpuid=0):
if src_vocab_info['changed'] or tgt_vocab_info['changed']:
model_path = checkpoint.update_vocab(
model_path,
os.path.join(self._output_dir, 'new_vocab_checkpoint'),
src_vocab_info['model'],
tgt_vocab_info['model'],
new_src_vocab=src_vocab_info['current']
if src_vocab_info['changed'] else None,
new_tgt_vocab=tgt_vocab_info['current']
if tgt_vocab_info['changed'] else None,
mode='replace',
session_config=tf.ConfigProto(device_count={'GPU': 0}))
model_dir, model = self._load_model(
model_type=config['options'].get('model_type'),
model_file=config['options'].get('model'),
model_path=model_path)
run_config = copy.deepcopy(config['options'].get('config', {}))
run_config['model_dir'] = model_dir
if 'data' not in run_config:
run_config['data'] = {}
if 'train' not in run_config:
run_config['train'] = {}
run_config['data']['source_words_vocabulary'] = src_vocab_info[
'current']
run_config['data']['target_words_vocabulary'] = tgt_vocab_info[
'current']
run_config['data']['train_features_file'] = src_file
run_config['data']['train_labels_file'] = tgt_file
if 'train_steps' not in run_config['train']:
run_config['train']['single_pass'] = True
run_config['train']['train_steps'] = None
if 'sample_buffer_size' not in run_config['train']:
run_config['train']['sample_buffer_size'] = -1
runner = onmt.Runner(model,
run_config,
num_devices=utils.count_devices(gpuid),
auto_config=config['options'].get(
'auto_config', False))
runner.train()
return self._list_model_files(model_dir)
def train(self, config, src_file, tgt_file, model_path=None, gpuid=0):
model_dir, model = self._load_model(
model_type=config['options'].get('model_type'),
model_file=config['options'].get('model'),
model_path=model_path)
run_config = copy.deepcopy(config['options']['config'])
run_config['model_dir'] = model_dir
for k, v in six.iteritems(run_config['data']):
run_config['data'][k] = self._convert_vocab(v)
run_config['data']['train_features_file'] = src_file
run_config['data']['train_labels_file'] = tgt_file
if 'train_steps' not in run_config['train']:
run_config['train']['single_pass'] = True
run_config['train']['train_steps'] = None
if 'sample_buffer_size' not in run_config['train']:
run_config['train']['sample_buffer_size'] = -1
onmt.Runner(model, run_config,
num_devices=utils.count_devices(gpuid)).train()
return self._list_model_files(model_dir)
def train(self,
config,
src_file,
tgt_file,
src_vocab_info,
tgt_vocab_info,
align_file=None,
model_path=None,
gpuid=0):
if model_path is None or tf.train.latest_checkpoint(
model_path) is None:
prev_src_vocab = None
prev_tgt_vocab = None
else:
prev_src_vocab = src_vocab_info.previous
prev_tgt_vocab = tgt_vocab_info.previous
runner = self._build_runner(config,
src_vocab=prev_src_vocab
or src_vocab_info.current,
tgt_vocab=prev_tgt_vocab
or tgt_vocab_info.current,
src_file=src_file,
tgt_file=tgt_file,
align_file=align_file,
model_path=model_path)
if prev_src_vocab or prev_tgt_vocab:
previous_model_dir = runner.model_dir
runner.update_vocab(
os.path.join(self._output_dir, 'new_vocab_checkpoint'),
src_vocab=src_vocab_info.current if prev_src_vocab else None,
tgt_vocab=tgt_vocab_info.current if prev_tgt_vocab else None)
shutil.rmtree(previous_model_dir)
output_dir, summary = runner.train(
num_devices=utils.count_devices(gpuid), return_summary=True)
return _list_checkpoint_files(output_dir), summary
