def _start_process(self):
if not os.path.isdir(self.engine.get_runtime_path()):
fileutils.makedirs(self.engine.get_runtime_path(), exist_ok=True)
self._log_file = self.engine.get_logfile(ClusterNode.__LOG_FILENAME, ensure=True)
args = ['-e', self.engine.name, '-p', str(self._cluster_ports[0]), str(self._cluster_ports[1]),
'--status-file', self._status_file]
if self._start_rest_server:
args.append('-a')
args.append(str(self._api_port))
if self._verbosity is not None:
args.append('-v')
args.append(str(self._verbosity))
if self._sibling is not None:
args.append('--member')
args.append(str(self._sibling))
env = os.environ.copy()
env['LD_LIBRARY_PATH'] = scripts.LIB_DIR
env['LC_ALL'] = 'en_US.UTF-8'
env['LANG'] = 'en_US.UTF-8'
command = mmt_javamain('eu.modernmt.cli.ClusterNodeMain', args,
hserr_path=os.path.abspath(os.path.join(self._log_file, os.pardir)))
log = open(self._log_file, 'wa')
if os.path.isfile(self._status_file):
os.remove(self._status_file)
return subprocess.Popen(command, stdout=open(os.devnull), stderr=log, shell=False, env=env)
def process(self,
source,
target,
input_paths,
output_path,
data_path=None):
args = ['-s', source, '-t', target, '--output', output_path, '--input']
for root in input_paths:
args.append(root)
if data_path is not None:
args.append('--dev')
args.append(
os.path.join(data_path, TrainingPreprocessor.DEV_FOLDER_NAME))
args.append('--test')
args.append(
os.path.join(data_path, TrainingPreprocessor.TEST_FOLDER_NAME))
command = mmt_javamain(self._java_mainclass, args)
shell.execute(command,
stdin=shell.DEVNULL,
stdout=shell.DEVNULL,
stderr=shell.DEVNULL)
return ParallelCorpus.splitlist(source, target, roots=output_path)
def __get_command(self, lang, print_tags, print_placeholders,
original_spacing):
args = ['--lang', lang]
if original_spacing:
args.append('--original-spacing')
if not print_tags:
args.append('--no-tags')
if print_placeholders:
args.append('--print-placeholders')
return mmt_javamain(self._java_mainclass, args)
def clean(self, source, target, input_paths, output_path):
args = ['-s', source, '-t', target, '--output', output_path, '--input']
for root in input_paths:
args.append(root)
command = mmt_javamain(self._java_mainclass, args)
shell.execute(command,
stdin=shell.DEVNULL,
stdout=shell.DEVNULL,
stderr=shell.DEVNULL)
return ParallelCorpus.splitlist(source, target, roots=output_path)[0]
def _start_process(self):
if not os.path.isdir(self.engine.get_runtime_path()):
fileutils.makedirs(self.engine.get_runtime_path(), exist_ok=True)
self._log_file = self.engine.get_logfile(ClusterNode.__LOG_FILENAME,
ensure=True)
args = [
'-e', self.engine.name, '-p',
str(self._cluster_ports[0]),
str(self._cluster_ports[1]), '--status-file', self._status_file
]
if self._start_rest_server:
args.append('-a')
args.append(str(self._api_port))
if self._verbosity is not None:
args.append('-v')
args.append(str(self._verbosity))
if self._sibling is not None:
args.append('--member')
args.append(str(self._sibling))
env = os.environ.copy()
env['LD_LIBRARY_PATH'] = scripts.LIB_DIR
env['LC_ALL'] = 'en_US.UTF-8'
env['LANG'] = 'en_US.UTF-8'
command = mmt_javamain('eu.modernmt.cli.ClusterNodeMain',
args,
hserr_path=os.path.abspath(
os.path.join(self._log_file, os.pardir)))
log = open(self._log_file, 'wa')
if os.path.isfile(self._status_file):
os.remove(self._status_file)
return subprocess.Popen(command,
stdout=open(os.devnull),
stderr=log,
shell=False,
env=env)
def create_index(self, corpora, lang, log_file=None):
source_paths = set()
for corpus in corpora:
source_paths.add(corpus.root)
fileutils.makedirs(self._index, exist_ok=True)
args = ['-l', lang, '-i', self._index, '-c']
for source_path in source_paths:
args.append(source_path)
command = mmt_javamain(self._java_mainclass, args)
log = shell.DEVNULL
try:
if log_file is not None:
log = open(log_file, 'w')
shell.execute(command, stdout=log, stderr=log)
finally:
if log_file is not None:
log.close()