def __init__(self,
name,
source_lang,
target_lang,
roots,
debug=False,
steps=None,
split_trainingset=True,
max_training_words=None):
EngineBuilder.__init__(
self, PhraseBasedEngine(name, source_lang, target_lang), roots,
debug, steps, split_trainingset, max_training_words)
def __init__(self, name, source_lang, target_lang, roots, debug=False, steps=None, split_trainingset=True,
validation_corpora=None, checkpoint=None, metadata=None, max_training_words=None, gpus=None,
training_args=None):
torch_setup(gpus=gpus, random_seed=3435)
self._training_opts = NMTEngineTrainer.Options()
if training_args is not None:
self._training_opts.load_from_dict(training_args.__dict__)
engine = NeuralEngine(name, source_lang, target_lang, bpe_symbols=self._training_opts.bpe_symbols,
max_vocab_size=self._training_opts.max_vocab_size,
vocab_pruning_threshold=self._training_opts.vocab_pruning_threshold)
EngineBuilder.__init__(self, engine, roots, debug, steps, split_trainingset, max_training_words)
self._valid_corpora_path = validation_corpora if validation_corpora is not None \
else os.path.join(self._engine.data_path, TrainingPreprocessor.DEV_FOLDER_NAME)
self._checkpoint = checkpoint
self._metadata = metadata
def _check_constraints(self):
free_space_on_disk = fileutils.df(self._engine.path)[2]
corpus_size_on_disk = 0
for root in self._roots:
corpus_size_on_disk += fileutils.du(root)
free_memory = fileutils.free()
recommended_mem = self._GB * corpus_size_on_disk / (
350 * self._MB) # 1G RAM every 350M on disk
recommended_disk = 10 * corpus_size_on_disk
if free_memory < recommended_mem or free_space_on_disk < recommended_disk:
if free_memory < recommended_mem:
raise EngineBuilder.HWConstraintViolated(
'more than %.fG of RAM recommended, only %.fG available' %
(recommended_mem / self._GB, free_memory / self._GB))
if free_space_on_disk < recommended_disk:
raise EngineBuilder.HWConstraintViolated(
'more than %.fG of storage recommended, only %.fG available'
% (recommended_disk / self._GB,
free_space_on_disk / self._GB))
def _check_constraints(self):
recommended_gpu_ram = 2 * self._GB
# Get the list of GPUs to employ using torch utils (This takes into account the user's choice)
gpus = torch_utils.torch_get_gpus()
if gpus is None or len(gpus) == 0:
raise EngineBuilder.HWConstraintViolated(
'No GPU for Neural engine training, the process will take very long time to complete.')
# AT THE MOMENT TRAINING IS MONOGPU AND WE ONLY USE THE FIRST AVAILABLE GPU FOR TRAINING.
# SO JUST CHECK CONSTRAINTS FOR IT. THIS MAY CHANGE IN THE FUTURE
gpus = [gpus[0]]
gpus_ram = self._get_gpus_ram(gpus)
for i in range(len(gpus_ram)):
if gpus_ram[i] < recommended_gpu_ram:
raise EngineBuilder.HWConstraintViolated(
'The RAM of GPU %d is only %.fG. More than %.fG of RAM recommended for each GPU.' %
(gpus[i], gpus_ram[i] / self._GB, recommended_gpu_ram / self._GB)
)
def _build_schedule(self):
return EngineBuilder._build_schedule(self) + [
self._train_tm, self._train_lm, self._write_moses_config
]
def _build_schedule(self):
return EngineBuilder._build_schedule(self) + \
[self._build_memory, self._prepare_training_data, self._train_decoder, self._merge_checkpoints]