def train(self, config, src_file, tgt_file, src_vocab_info, tgt_vocab_info, align_file=None, model_path=None, gpuid=0): if src_vocab_info['changed'] or tgt_vocab_info['changed']: model_path = checkpoint.update_vocab( model_path, os.path.join(self._output_dir, 'new_vocab_checkpoint'), src_vocab_info['model'], tgt_vocab_info['model'], new_src_vocab=src_vocab_info['current'] if src_vocab_info['changed'] else None, new_tgt_vocab=tgt_vocab_info['current'] if tgt_vocab_info['changed'] else None, mode='replace', session_config=tf.ConfigProto(device_count={'GPU': 0})) model_dir, model = self._load_model( model_type=config['options'].get('model_type'), model_file=config['options'].get('model'), model_path=model_path) run_config = copy.deepcopy(config['options'].get('config', {})) run_config['model_dir'] = model_dir if 'data' not in run_config: run_config['data'] = {} if 'train' not in run_config: run_config['train'] = {} run_config['data']['source_words_vocabulary'] = src_vocab_info[ 'current'] run_config['data']['target_words_vocabulary'] = tgt_vocab_info[ 'current'] run_config['data']['train_features_file'] = src_file run_config['data']['train_labels_file'] = tgt_file if align_file is not None and os.path.exists(align_file): run_config['data']['train_alignments'] = align_file if "params" not in run_config: run_config["params"] = {} if "guided_alignment_type" not in run_config["params"]: run_config["params"]["guided_alignment_type"] = "ce" if 'train_steps' not in run_config['train']: run_config['train']['single_pass'] = True run_config['train']['train_steps'] = None if 'sample_buffer_size' not in run_config['train']: run_config['train']['sample_buffer_size'] = -1 if 'average_last_checkpoints' not in run_config['train']: run_config['train']['average_last_checkpoints'] = 0 runner = onmt.Runner(model, run_config, num_devices=utils.count_devices(gpuid), auto_config=config['options'].get( 'auto_config', False)) output_dir = runner.train() if output_dir != model_dir: shutil.copy(os.path.join(model_dir, "model_description.py"), output_dir) return self._list_model_files(output_dir)
def train( self, config, src_file, tgt_file, src_vocab_info, tgt_vocab_info, align_file=None, example_weights_file=None, model_path=None, gpuid=0, ): if model_path is None or tf.train.latest_checkpoint( model_path) is None: prev_src_vocab = None prev_tgt_vocab = None else: prev_src_vocab = src_vocab_info.previous prev_tgt_vocab = tgt_vocab_info.previous runner = self._build_runner( config, src_vocab=prev_src_vocab or src_vocab_info.current, tgt_vocab=prev_tgt_vocab or tgt_vocab_info.current, src_file=src_file, tgt_file=tgt_file, align_file=align_file, example_weights_file=example_weights_file, model_path=model_path, ) if prev_src_vocab or prev_tgt_vocab: previous_model_dir = runner.model_dir # Update the model vocabulary on CPU to avoid initializing the GPU context # and allow batch size autotuning to run properly afterwards. with tf.device("cpu"): runner.update_vocab( os.path.join(self._output_dir, "new_vocab_checkpoint"), src_vocab=src_vocab_info.current if prev_src_vocab else None, tgt_vocab=tgt_vocab_info.current if prev_tgt_vocab else None, ) shutil.rmtree(previous_model_dir) output_dir, summary = runner.train( num_devices=utils.count_devices(gpuid), return_summary=True, fallback_to_cpu=False, ) return _list_checkpoint_files(output_dir), summary
def train(self, config, src_file, tgt_file, src_vocab_info, tgt_vocab_info, model_path=None, gpuid=0): if src_vocab_info['changed'] or tgt_vocab_info['changed']: model_path = checkpoint.update_vocab( model_path, os.path.join(self._output_dir, 'new_vocab_checkpoint'), src_vocab_info['model'], tgt_vocab_info['model'], new_src_vocab=src_vocab_info['current'] if src_vocab_info['changed'] else None, new_tgt_vocab=tgt_vocab_info['current'] if tgt_vocab_info['changed'] else None, mode='replace', session_config=tf.ConfigProto(device_count={'GPU': 0})) model_dir, model = self._load_model( model_type=config['options'].get('model_type'), model_file=config['options'].get('model'), model_path=model_path) run_config = copy.deepcopy(config['options'].get('config', {})) run_config['model_dir'] = model_dir if 'data' not in run_config: run_config['data'] = {} if 'train' not in run_config: run_config['train'] = {} run_config['data']['source_words_vocabulary'] = src_vocab_info[ 'current'] run_config['data']['target_words_vocabulary'] = tgt_vocab_info[ 'current'] run_config['data']['train_features_file'] = src_file run_config['data']['train_labels_file'] = tgt_file if 'train_steps' not in run_config['train']: run_config['train']['single_pass'] = True run_config['train']['train_steps'] = None if 'sample_buffer_size' not in run_config['train']: run_config['train']['sample_buffer_size'] = -1 runner = onmt.Runner(model, run_config, num_devices=utils.count_devices(gpuid), auto_config=config['options'].get( 'auto_config', False)) runner.train() return self._list_model_files(model_dir)
def train(self, config, src_file, tgt_file, model_path=None, gpuid=0): model_dir, model = self._load_model( model_type=config['options'].get('model_type'), model_file=config['options'].get('model'), model_path=model_path) run_config = copy.deepcopy(config['options']['config']) run_config['model_dir'] = model_dir for k, v in six.iteritems(run_config['data']): run_config['data'][k] = self._convert_vocab(v) run_config['data']['train_features_file'] = src_file run_config['data']['train_labels_file'] = tgt_file if 'train_steps' not in run_config['train']: run_config['train']['single_pass'] = True run_config['train']['train_steps'] = None if 'sample_buffer_size' not in run_config['train']: run_config['train']['sample_buffer_size'] = -1 onmt.Runner(model, run_config, num_devices=utils.count_devices(gpuid)).train() return self._list_model_files(model_dir)
def train(self, config, src_file, tgt_file, src_vocab_info, tgt_vocab_info, align_file=None, model_path=None, gpuid=0): if model_path is None or tf.train.latest_checkpoint( model_path) is None: prev_src_vocab = None prev_tgt_vocab = None else: prev_src_vocab = src_vocab_info.previous prev_tgt_vocab = tgt_vocab_info.previous runner = self._build_runner(config, src_vocab=prev_src_vocab or src_vocab_info.current, tgt_vocab=prev_tgt_vocab or tgt_vocab_info.current, src_file=src_file, tgt_file=tgt_file, align_file=align_file, model_path=model_path) if prev_src_vocab or prev_tgt_vocab: previous_model_dir = runner.model_dir runner.update_vocab( os.path.join(self._output_dir, 'new_vocab_checkpoint'), src_vocab=src_vocab_info.current if prev_src_vocab else None, tgt_vocab=tgt_vocab_info.current if prev_tgt_vocab else None) shutil.rmtree(previous_model_dir) output_dir, summary = runner.train( num_devices=utils.count_devices(gpuid), return_summary=True) return _list_checkpoint_files(output_dir), summary