def __test(self, src, tgt, filename):
src_file = os.path.join(TEST_RESOURCES, 'tag_projection_dataset',
filename + '.' + src)
tgt_file = os.path.join(TEST_RESOURCES, 'tag_projection_dataset',
filename + '.' + tgt)
alg_file = os.path.join(TEST_RESOURCES, 'tag_projection_dataset',
filename + '.alg')
if not os.path.isfile(src_file) or not os.path.isfile(
tgt_file) or not os.path.isfile(alg_file):
self.skipTest("external resource not available")
java_cmd = mmt_java(
'eu.modernmt.processing.tags.cli.XMLProjectorTestMain',
[src_file, tgt_file, alg_file])
with tempfile.NamedTemporaryFile() as out_stream:
osutils.shell_exec(java_cmd, stdout=out_stream)
out_stream.flush()
with _Reader(src_file, out_stream.name, alg_file) as reader:
for src_line, tgt_line, alg_line in reader:
src_line, tgt_line = src_line.rstrip(), tgt_line.rstrip()
src_tags, tgt_tags = self._extract_tags(
src_line), self._extract_tags(tgt_line)
if set(src_tags) != set(tgt_tags):
self.fail(
'Not all tags were projected:\n\t%s\n\t%s\n\t%s' %
(src_line, tgt_line, alg_line))
if not self.__validate_tags(tgt_tags):
self.fail('Invalid tag projection:\n\t%s\n\t%s\n\t%s' %
(src_line, tgt_line, alg_line))
def start(self, api_port=None, cluster_port=None, binlog_port=None,
db_port=None, leader=None, verbosity=None, remote_debug=False, log_file=None):
if log_file is not None:
self._log_file = log_file
if not os.path.isdir(self.engine.runtime_path):
os.makedirs(self.engine.runtime_path)
args = ['-e', self.engine.name, '--status-file', self._status_file, '--log-file', self._log_file]
if cluster_port is not None:
args.append('--cluster-port')
args.append(str(cluster_port))
if api_port is not None:
args.append('--api-port')
args.append(str(api_port))
if binlog_port is not None:
args.append('--binlog-port')
args.append(str(binlog_port))
if db_port is not None:
args.append('--db-port')
args.append(str(db_port))
if verbosity is not None:
args.append('-v')
args.append(str(verbosity))
if leader is not None:
args.append('--leader')
args.append(leader)
# read memory size
mem_bytes = os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES') # e.g. 4015976448
mem_mb = mem_bytes / (1024. ** 2) # e.g. 3.74
heap_mb = max(min(mem_mb / 4, 16 * 1024), 1024)
heap_mb = int(heap_mb / 1024) * 1024
logs_folder = os.path.abspath(os.path.join(self._log_file, os.pardir))
command = mmt_java('eu.modernmt.cli.ClusterNodeMain', args, logs_path=logs_folder, remote_debug=remote_debug,
max_heap_mb=heap_mb, server=True)
if os.path.isfile(self._status_file):
os.remove(self._status_file)
if not super()._start(command):
raise Exception('failed to start node, check log file for more details: %s' % self._log_file)
def start(self):
command = mmt_java(
'eu.modernmt.cli.BackupDaemonMain',
args=['-e', self.engine.name, '-i', '3600', '-l', '1'])
env = dict(os.environ, MMT_Q_HOST=network.get_ip())
self._process = osutils.shell_exec(command, background=True, env=env)