def load(name): if os.sep in name: raise IllegalArgumentException('Invalid engine name: "%s"' % name) config_path = Engine._get_config_path(name) if not os.path.isfile(config_path): raise IllegalArgumentException("Engine '%s' not found" % name) # parse the source language and target language from the configuration file def _get_child(root, child_name): elements = root.getElementsByTagName(child_name) return elements[0] if len(elements) > 0 else None languages = [] config_root = minidom.parse(config_path).documentElement engine_el = _get_child(config_root, 'engine') lang_el = _get_child(engine_el, 'languages') if lang_el is not None: for pair_el in lang_el.getElementsByTagName('pair'): source_lang = pair_el.getAttribute('source') target_lang = pair_el.getAttribute('target') languages.append((source_lang, target_lang)) else: source_lang = engine_el.getAttribute('source-language') target_lang = engine_el.getAttribute('target-language') languages.append((source_lang, target_lang)) return Engine(name, languages)
def get_memory_id_by_name(self, name): try: return int(name) except ValueError: memories = self.api.get_all_memories() ids = [m['id'] for m in memories if m['name'] == name] if len(ids) == 0: raise IllegalArgumentException('unable to find memory "%s"' % name) elif len(ids) > 1: raise IllegalArgumentException( 'ambiguous memory name "%s", choose one of the following ids: %s' % (name, str(ids))) else: return ids[0]
def __init__(self, name, source_lang, target_lang, roots, debug=False, steps=None, split_trainingset=True): # the builder already prepares a basic, "empty" engine self._engine = MMTEngine(name, source_lang, target_lang) self._roots = roots self._debug = debug self._split_trainingset = split_trainingset self._temp_dir = None self._checkpoint_path = None self._scheduled_steps = None self._passed_steps = None # if no steps are passed, all training steps must be performed. # else, only perform the passed training steps (if they are all legal) if steps is None: self._scheduled_steps = self.DEFAULT_TRAINING_STEPS else: unknown_steps = [ step for step in steps if step not in self.DEFAULT_TRAINING_STEPS ] if len(unknown_steps) > 0: raise IllegalArgumentException('Unknown training steps: ' + str(unknown_steps)) self._scheduled_steps = steps print
def load(name): # figure the configuration file path from the engine name config_path = Engine._get_config_path(name) if not os.path.isfile(config_path): raise IllegalArgumentException("Engine '%s' not found" % name) # parse the source language and target language from the configuration file engine_el = minidom.parse( config_path).documentElement.getElementsByTagName("engine")[0] engine_type = engine_el.getAttribute('type') source_lang = engine_el.getAttribute('source-language') target_lang = engine_el.getAttribute('target-language') # create and return a new engine with that name, source target and language target if engine_type == 'neural': from cli.mmt.neural import NeuralEngine return NeuralEngine(name, source_lang, target_lang, bpe_symbols=None) else: from cli.mmt.phrasebased import PhraseBasedEngine return PhraseBasedEngine(name, source_lang, target_lang)
def import_corpus(self, domain_id, corpus, callback=None, refresh_rate_in_seconds=1): if type(corpus) == TMXCorpus: job = self.api.import_into_domain(domain_id, tmx=corpus.get_tmx()) elif type(corpus) == FileParallelCorpus: source_file = corpus.get_file(self.engine.source_lang) target_file = corpus.get_file(self.engine.target_lang) job = self.api.import_into_domain(domain_id, source_file=source_file, target_file=target_file) else: raise IllegalArgumentException('Invalid corpus type: ' + str(type(corpus))) if callback is not None: callback(job) while job['progress'] != 1.0: time.sleep(refresh_rate_in_seconds) job = self.api.get_import_job(job['id']) if callback is not None: callback(job)
def nmt_tune(self, corpora, debug=False, listener=None, max_lines=None, lr_delta=0.1, max_epochs=10, gpus=None): target_lang = self.engine.target_lang source_lang = self.engine.source_lang corpora = [ corpus for corpus in corpora if source_lang in corpus.langs and target_lang in corpus.langs ] if len(corpora) == 0: raise IllegalArgumentException( 'No %s > %s corpora found into specified path' % (source_lang, target_lang)) if listener is None: listener = self.TuneListener() listener.on_tuning_begin(corpora, self, 2) working_dir = self.engine.get_tempdir('tuning') log_file = self.engine.get_logfile('nmt_tune') try: content = [] with listener.step('Corpora pre-processing'): validation_corpora_path = os.path.join(working_dir, 'valid_set') validation_corpora, _ = self.engine.training_preprocessor.process( corpora, validation_corpora_path) for corpus in validation_corpora: with corpus.reader([source_lang, target_lang]) as reader: for source, target in reader: content.append((source.strip(), target.strip())) if 0 < max_lines < len(content): random.shuffle(content) content = content[:max_lines] with listener.step('Tuning'): bleu_score = self.engine.tune(content, working_dir, lr_delta=lr_delta, max_epochs=max_epochs, log_file=log_file, gpus=gpus) listener.on_tuning_end(self, bleu_score) finally: if not debug: self.engine.clear_tempdir("tuning")
def rename_domain(self, domain, name): try: domain = int(domain) except ValueError: domains = self.api.get_all_domains() ids = [d['id'] for d in domains if d['name'] == domain] if len(ids) == 0: raise IllegalArgumentException('unable to find domain "' + domain + '"') elif len(ids) > 1: raise IllegalArgumentException( 'ambiguous domain name "' + domain + '", choose one of the following ids: ' + str(ids)) else: domain = ids[0] return self.api.rename_domain(domain, name)
def __parse_context_map(text): context = [] try: for score in text.split(','): name, value = score.split(':', 2) value = float(value) context.append({'id': name, 'score': value}) except ValueError: raise IllegalArgumentException('invalid context weights map: ' + text) return context
def __init__(self, plan, filtered_steps=None): self._plan = plan self._passed_steps = [] all_steps = self.all_steps() if filtered_steps is not None: self._scheduled_steps = filtered_steps unknown_steps = [step for step in self._scheduled_steps if step not in all_steps] if len(unknown_steps) > 0: raise IllegalArgumentException('Unknown training steps: ' + str(unknown_steps)) else: self._scheduled_steps = all_steps
def __parse_context_vector(text): context = [] try: for score in text.split(','): id, value = score.split(':', 2) value = float(value) context.append({'domain': int(id), 'score': value}) except ValueError: raise IllegalArgumentException('invalid context weights map: ' + text) return context
def load(name): # figure the configuration file path from the engine name config_path = MMTEngine._get_config_path(name) if not os.path.isfile(config_path): raise IllegalArgumentException("Engine '%s' not found" % name) # parse the source language and target language from the configuration file engine_el = minidom.parse( config_path).documentElement.getElementsByTagName("engine")[0] source_lang = engine_el.getAttribute('source-language') target_lang = engine_el.getAttribute('target-language') # create and return a new engine with that name, source target and language target return MMTEngine(name, source_lang, target_lang)
def create_domain(self, tmx=None, source_file=None, target_file=None, name=None): if source_file is not None and target_file is not None: params = { 'source_local_file': source_file, 'target_local_file': target_file } elif tmx is not None: params = {'tmx_local_file': tmx} else: raise IllegalArgumentException('missing corpus for domain') if name is not None: params['name'] = name return self._post('domains', params=params)
def build(self, roots, debug=False, steps=None, split_trainingset=True): self._temp_dir = self._engine.get_tempdir('training', ensure=True) source_lang = self._engine.source_lang target_lang = self._engine.target_lang bilingual_corpora, monolingual_corpora = BilingualCorpus.splitlist( source_lang, target_lang, roots=roots) if len(bilingual_corpora) == 0: raise IllegalArgumentException( 'you project does not include %s-%s data.' % (source_lang.upper(), target_lang.upper())) if steps is None: steps = self._engine.training_steps else: unknown_steps = [ step for step in steps if step not in self._engine.training_steps ] if len(unknown_steps) > 0: raise IllegalArgumentException('Unknown training steps: ' + str(unknown_steps)) shutil.rmtree(self._engine.path, ignore_errors=True) os.makedirs(self._engine.path) # Check disk space constraints free_space_on_disk = fileutils.df(self._engine.path)[2] corpus_size_on_disk = 0 for root in roots: corpus_size_on_disk += fileutils.du(root) free_memory = fileutils.free() recommended_mem = self.__GB * corpus_size_on_disk / ( 350 * self.__MB) # 1G RAM every 350M on disk recommended_disk = 10 * corpus_size_on_disk if free_memory < recommended_mem or free_space_on_disk < recommended_disk: if free_memory < recommended_mem: print '> WARNING: more than %.fG of RAM recommended, only %.fG available' % \ (recommended_mem / self.__GB, free_memory / self.__GB) if free_space_on_disk < recommended_disk: print '> WARNING: more than %.fG of storage recommended, only %.fG available' % \ (recommended_disk / self.__GB, free_space_on_disk / self.__GB) print logger = _builder_logger( len(steps) + 1, self._engine.get_logfile('training')) try: logger.start(self._engine, bilingual_corpora, monolingual_corpora) unprocessed_bicorpora = bilingual_corpora unprocessed_monocorpora = monolingual_corpora # TM draft-translations cleanup if 'tm_cleanup' in steps: with logger.step('TMs clean-up') as _: unprocessed_bicorpora = self._engine.cleaner.clean( unprocessed_bicorpora, self._get_tempdir('clean_tms'), log=logger.stream) cleaned_bicorpora = unprocessed_bicorpora processed_bicorpora = unprocessed_bicorpora processed_monocorpora = unprocessed_monocorpora # Preprocessing if 'preprocess' in steps: with logger.step('Corpora preprocessing') as _: unprocessed_bicorpora, unprocessed_monocorpora = self._engine.db.generate( unprocessed_bicorpora, unprocessed_monocorpora, self._get_tempdir('training_corpora'), log=logger.stream) processed_bicorpora, processed_monocorpora = self._engine.training_preprocessor.process( unprocessed_bicorpora + unprocessed_monocorpora, self._get_tempdir('preprocessed'), (self._engine.data_path if split_trainingset else None), log=logger.stream) cleaned_bicorpora = self._engine.training_preprocessor.clean( processed_bicorpora, self._get_tempdir('clean_corpora')) # Training Context Analyzer if 'context_analyzer' in steps: with logger.step('Context Analyzer training') as _: self._engine.analyzer.create_index(unprocessed_bicorpora, log=logger.stream) # Aligner if 'aligner' in steps: with logger.step('Aligner training') as _: working_dir = self._get_tempdir('aligner') self._engine.aligner.build(cleaned_bicorpora, working_dir, log=logger.stream) # Training Translation Model if 'tm' in steps: with logger.step('Translation Model training') as _: working_dir = self._get_tempdir('tm') self._engine.pt.train(cleaned_bicorpora, self._engine.aligner, working_dir, log=logger.stream) # Training Adaptive Language Model if 'lm' in steps: with logger.step('Language Model training') as _: working_dir = self._get_tempdir('lm') self._engine.lm.train(processed_bicorpora + processed_monocorpora, target_lang, working_dir, log=logger.stream) # Writing config file with logger.step('Writing config files') as _: self._engine.write_configs() logger.completed() except: logger.error() raise finally: logger.close() if not debug: self._engine.clear_tempdir('training')
def tune(self, corpora=None, debug=False, context_enabled=True, random_seeds=False, max_iterations=25, early_stopping_value=None): if corpora is None: corpora = BilingualCorpus.list( os.path.join(self.engine.data_path, TrainingPreprocessor.DEV_FOLDER_NAME)) target_lang = self.engine.target_lang source_lang = self.engine.source_lang corpora = [ corpus for corpus in corpora if source_lang in corpus.langs and target_lang in corpus.langs ] if len(corpora) == 0: raise IllegalArgumentException( 'No %s > %s corpora found into specified path' % (source_lang, target_lang)) source_corpora = [ BilingualCorpus.make_parallel(corpus.name, corpus.get_folder(), [source_lang]) for corpus in corpora ] reference_corpora = [ BilingualCorpus.make_parallel(corpus.name, corpus.get_folder(), [target_lang]) for corpus in corpora ] cmdlogger = _tuning_logger(4) cmdlogger.start(self, corpora) working_dir = self.engine.get_tempdir('tuning') mert_wd = os.path.join(working_dir, 'mert') try: # Tokenization tokenizer = Tokenizer(target_lang) tokenized_output = os.path.join(working_dir, 'reference_corpora') fileutils.makedirs(tokenized_output, exist_ok=True) with cmdlogger.step('Corpora tokenization') as _: reference_corpora = tokenizer.process_corpora( reference_corpora, tokenized_output) # Create merged corpus with cmdlogger.step('Merging corpus') as _: # source source_merged_corpus = os.path.join(working_dir, 'corpus.' + source_lang) with open(source_merged_corpus, 'wb') as out: for corpus in source_corpora: out.write(corpus.get_file(source_lang) + '\n') # target target_merged_corpus = os.path.join(working_dir, 'corpus.' + target_lang) fileutils.merge([ corpus.get_file(target_lang) for corpus in reference_corpora ], target_merged_corpus) # Run MERT algorithm with cmdlogger.step('Tuning') as _: # Start MERT decoder_flags = ['--port', str(self.api.port)] if self.api.root is not None: decoder_flags += ['--root', self.api.root] if not context_enabled: decoder_flags.append('--skip-context-analysis') decoder_flags.append('1') fileutils.makedirs(mert_wd, exist_ok=True) with tempfile.NamedTemporaryFile() as runtime_moses_ini: command = [ self._mert_script, source_merged_corpus, target_merged_corpus, self._mert_i_script, runtime_moses_ini.name, '--threads', str(multiprocessing.cpu_count()), '--mertdir', cli.BIN_DIR, '--mertargs', '\'--binary --sctype BLEU\'', '--working-dir', mert_wd, '--nbest', '100', '--decoder-flags', '"' + ' '.join(decoder_flags) + '"', '--nonorm', '--closest', '--no-filter-phrase-table' ] if early_stopping_value is not None: command += [ '--bleuscorer', self._scorer_script, '--bleuscorer-flags "-nt" --early-stopping-value %d' % early_stopping_value ] if not random_seeds: command.append('--predictable-seeds') if max_iterations > 0: command.append('--maximum-iterations={num}'.format( num=max_iterations)) with open(self.engine.get_logfile('mert'), 'wb') as log: shell.execute(' '.join(command), stdout=log, stderr=log) # Read optimized configuration with cmdlogger.step('Applying changes') as _: bleu_score = 0 weights = {} found_weights = False with open(os.path.join(mert_wd, 'moses.ini')) as moses_ini: for line in moses_ini: line = line.strip() if len(line) == 0: continue elif found_weights: tokens = line.split() weights[tokens[0].rstrip('=')] = [ float(val) for val in tokens[1:] ] elif line.startswith('# BLEU'): bleu_score = float(line.split()[2]) elif line == '[weight]': found_weights = True _ = self.api.update_features(weights) cmdlogger.completed(bleu_score) finally: if not debug: self.engine.clear_tempdir("tuning")
def translate(self, corpora, dest_path=None, debug=False): if len(corpora) == 0: raise IllegalArgumentException('empty corpora') if dest_path: fileutils.makedirs(dest_path, exist_ok=True) target_lang = self._engine.target_lang source_lang = self._engine.source_lang working_dir = self._engine.get_tempdir('evaluation') have_references = False try: results = [] # Process references corpora_path = os.path.join(working_dir, 'corpora') corpora = self._xmlencoder.encode(corpora, corpora_path) reference = os.path.join(working_dir, 'reference.' + target_lang) source = os.path.join(working_dir, 'source.' + source_lang) refs = [corpus.get_file(target_lang) for corpus in corpora if corpus.get_file(target_lang)] have_references = len(refs) > 0 fileutils.merge(refs, reference) # tolerates missing reference fileutils.merge([corpus.get_file(source_lang) for corpus in corpora], source) if dest_path: for corpus in corpora: corpus.copy(dest_path, suffixes={source_lang: '.src', target_lang: '.ref', 'tmx': '.src'}) # Translate translator = self._translator name = translator.name() result = _EvaluationResult(translator) results.append(result) translations_path = os.path.join(working_dir, 'translations', result.id + '.raw') xmltranslations_path = os.path.join(working_dir, 'translations', result.id) fileutils.makedirs(translations_path, exist_ok=True) try: translated, mtt, parallelism = translator.translate(corpora, translations_path) filename = result.id + '.' + target_lang result.mtt = mtt result.parallelism = parallelism result.translated_corpora = self._xmlencoder.encode(translated, xmltranslations_path) result.merge = os.path.join(working_dir, filename) fileutils.merge([corpus.get_file(target_lang) for corpus in result.translated_corpora], result.merge) if dest_path: for corpus in result.translated_corpora: corpus.copy(dest_path, suffixes={target_lang: '.hyp', 'tmx': '.hyp'}) except TranslateError as e: result.error = e except Exception as e: result.error = TranslateError('Unexpected ERROR: ' + str(e.message)) if result.error is None: if have_references: scorer = BLEUScore() # bleu in range [0;1) bleu = scorer.calculate(result.merge, reference) return bleu else: return True else: print(result.error) return None finally: if not debug: self._engine.clear_tempdir('evaluation')
def evaluate(self, corpora, heval_output=None, debug=False): target_lang = self._engine.target_lang source_lang = self._engine.source_lang corpora = [corpus for corpus in corpora if source_lang in corpus.langs and target_lang in corpus.langs] if len(corpora) == 0: raise IllegalArgumentException('No %s > %s corpora found into specified path' % (source_lang, target_lang)) if heval_output is not None: fileutils.makedirs(heval_output, exist_ok=True) logger = _evaluate_logger() logger.start(corpora) working_dir = self._engine.get_tempdir('evaluation') try: results = [] # Process references with logger.step('Preparing corpora') as _: corpora_path = os.path.join(working_dir, 'corpora') corpora = self._xmlencoder.encode(corpora, corpora_path) reference = os.path.join(working_dir, 'reference.' + target_lang) source = os.path.join(working_dir, 'source.' + source_lang) fileutils.merge([corpus.get_file(target_lang) for corpus in corpora], reference) fileutils.merge([corpus.get_file(source_lang) for corpus in corpora], source) if heval_output is not None: self._heval_outputter.write(lang=target_lang, input_file=reference, output_file=os.path.join(heval_output, 'reference.' + target_lang)) self._heval_outputter.write(lang=source_lang, input_file=source, output_file=os.path.join(heval_output, 'source.' + source_lang)) # Translate for translator in self._translators: name = translator.name() with logger.step('Translating with %s' % name) as _: result = _EvaluationResult(translator) results.append(result) translations_path = os.path.join(working_dir, 'translations', result.id + '.raw') xmltranslations_path = os.path.join(working_dir, 'translations', result.id) fileutils.makedirs(translations_path, exist_ok=True) try: translated, mtt, parallelism = translator.translate(corpora, translations_path) filename = result.id + '.' + target_lang result.mtt = mtt result.parallelism = parallelism result.translated_corpora = self._xmlencoder.encode(translated, xmltranslations_path) result.merge = os.path.join(working_dir, filename) fileutils.merge([corpus.get_file(target_lang) for corpus in result.translated_corpora], result.merge) if heval_output is not None: self._heval_outputter.write(lang=target_lang, input_file=result.merge, output_file=os.path.join(heval_output, filename)) except TranslateError as e: result.error = e except Exception as e: result.error = TranslateError('Unexpected ERROR: ' + str(e.message)) # Check corpora length reference_lines = fileutils.linecount(reference) for result in results: if result.error is not None: continue lines = fileutils.linecount(result.merge) if lines != reference_lines: raise TranslateError('Invalid line count for translator %s: expected %d, found %d.' % (result.translator.name(), reference_lines, lines)) # Scoring scorers = [(MatecatScore(), 'pes'), (BLEUScore(), 'bleu')] for scorer, field in scorers: with logger.step('Calculating %s' % scorer.name()) as _: for result in results: if result.error is not None: continue setattr(result, field, scorer.calculate(result.merge, reference)) logger.completed(results, scorers) return results finally: if not debug: self._engine.clear_tempdir('evaluation')
def _build(self, resume=False): self._temp_dir = self._engine.get_tempdir('training', ensure=(not resume)) self._checkpoint_path = os.path.join(self._temp_dir, 'checkpoint.json') self._passed_steps = [] if resume: self.load_checkpoint() else: self.save_checkpoint() # initialize thee checkpoint manager source_lang = self._engine.source_lang target_lang = self._engine.target_lang # separate bilingual and monolingual corpora in separate lists, reading them from roots bilingual_corpora, monolingual_corpora = BilingualCorpus.splitlist( source_lang, target_lang, roots=self._roots) # if no bilingual corpora are found, it is not possible to train the translation system if len(bilingual_corpora) == 0: raise IllegalArgumentException( 'you project does not include %s-%s data.' % (source_lang.upper(), target_lang.upper())) # if no old engines (i.e. engine folders) can be found, create a new one from scratch # if we are not trying to resume an old one, create from scratch anyway if not os.path.isdir(self._engine.path) or not resume: shutil.rmtree(self._engine.path, ignore_errors=True) os.makedirs(self._engine.path) # Check if all requirements are fulfilled before launching engine training self._check_constraints() # Create a new logger for the building activities, # passing it the amount of steps to perform (plus a non user-decidable step) # and the name of the log file to create logger = _builder_logger( len(self._scheduled_steps) + 1, self._engine.get_logfile('training')) delete_on_exit = not self._debug # Start the engine building (training) phases try: # tell the logger that the engine training has started logger.start(self._engine, bilingual_corpora, monolingual_corpora) # ~~~~~~~~~~~~~~~~~~~~~ RUN ALL STEPS ~~~~~~~~~~~~~~~~~~~~~ # Note: if resume is true, a step is only run if it was not in the previous attempt # run tm_cleanup step on the bilingual_corpora if required. # Obtain cleaned bicorpora cleaned_bicorpora = self._run_step('tm_cleanup', self._step_tm_cleanup, logger=logger, values=[bilingual_corpora], delete_on_exit=delete_on_exit) # run __db_map step (always: user can't skip it) # on the cleaned bicorpora and the original monocorpora; # obtain base bicorpora and base monocorpora base_bicorpora, base_monocorpora = self._run_step( '__db_map', self._step_init, forced=True, values=[cleaned_bicorpora, monolingual_corpora], delete_on_exit=delete_on_exit) # run preprocess step if required. # Return processed bi and mono corpora and cleaned bicorpora processed_bicorpora, processed_monocorpora, cleaned_bicorpora = \ self._run_step('preprocess', self._step_preprocess, logger=logger, values=[base_bicorpora, base_monocorpora, base_bicorpora], delete_on_exit=delete_on_exit) # run context_analyzer step base_bicorpora if required. _ = self._run_step('context_analyzer', self._step_context_analyzer, logger=logger, values=[base_bicorpora], delete_on_exit=delete_on_exit) # run aligner step cleaned_bicorpora if required. _ = self._run_step('aligner', self._step_aligner, logger=logger, values=[cleaned_bicorpora], delete_on_exit=delete_on_exit) # run tm step cleaned_bicorpora if required. _ = self._run_step('tm', self._step_tm, logger=logger, values=[cleaned_bicorpora], delete_on_exit=delete_on_exit) # run lm step on the joint list of processed_bicorpora and processed_monocorpora _ = self._run_step( 'lm', self._step_lm, logger=logger, values=[processed_bicorpora + processed_monocorpora], delete_on_exit=delete_on_exit) # Writing config file with logger.step('Writing config files') as _: self._engine.write_configs() # tell the logger that the engine training has completed logger.completed() # if this is not debug mode, then the training temporary folder must be deleted if not self._debug: self._engine.clear_tempdir('training') except: logger.error() raise finally: logger.close()
def evaluate(self, corpora, heval_output=None, debug=False): corpora = [ corpus for corpus in corpora if self._source_lang in corpus.langs and self._target_lang in corpus.langs ] if len(corpora) == 0: raise IllegalArgumentException( 'No %s > %s corpora found into specified path' % (self._source_lang, self._target_lang)) print '\n============== EVALUATION ==============\n' print 'Testing on %d lines:\n' % sum( [corpus.count_lines() for corpus in corpora]) if heval_output is not None: osutils.makedirs(heval_output, exist_ok=True) step_logger = _StepLogger() human_eval_outputter = HumanEvaluationFileOutputter( ) if heval_output is not None else None working_dir = self._engine.get_tempdir('evaluation') try: # Process references with step_logger.step('Preparing corpora') as _: source = os.path.join(working_dir, 'source.' + self._source_lang) osutils.concat( [corpus.get_file(self._source_lang) for corpus in corpora], source) reference = os.path.join(working_dir, 'reference.' + self._target_lang) osutils.concat( [corpus.get_file(self._target_lang) for corpus in corpora], reference + '.tmp') XMLEncoder().encode_file(reference + '.tmp', reference) os.remove(reference + '.tmp') if human_eval_outputter is not None: human_eval_outputter.write(lang=self._target_lang, input_file=reference, output_file=os.path.join( heval_output, 'reference.' + self._target_lang)) human_eval_outputter.write(lang=self._source_lang, input_file=source, output_file=os.path.join( heval_output, 'source.' + self._source_lang)) total_line_count = osutils.lc(reference) # Translate entries = [] for translator in self._translators: with step_logger.step('Translating with %s' % translator.name) as _: entry = self._translate_with(translator, corpora, working_dir, total_line_count) entries.append(entry) if entry.error is None and human_eval_outputter is not None: human_eval_file = os.path.join( heval_output, os.path.basename(entry.translation_file)) human_eval_outputter.write( lang=self._target_lang, input_file=entry.translation_file, output_file=human_eval_file) # Scoring for scorer in self._scorers: with step_logger.step('Calculating %s' % scorer.name()) as _: for entry in entries: if entry.error is not None: continue try: entry.scores[scorer] = scorer.calculate( entry.translation_file, reference) except Exception as e: entry.scores[scorer] = str(e) # Print results print '\n=============== RESULTS ================\n' for scorer in self._scorers: print scorer.name() + ':' for i, entry in enumerate( sorted(entries, key=lambda x: x.scores[scorer] if x.error is None else 0, reverse=True)): if entry.error is None: value = entry.scores[scorer] if isinstance(value, basestring): text = value else: text = '%.2f' % (value * 100) if i == 0: text += ' (Winner)' else: text = str(entry.error) print ' %s: %s' % (entry.translator.name.ljust(20), text) print print 'Translation Speed:' for entry in sorted(entries, key=lambda x: x.translation_time if x.error is None else float('inf')): if entry.error is None: text = '%.2fs per sentence' % entry.translation_time else: text = str(entry.error) print ' %s: %s' % (entry.translator.name.ljust(20), text) print finally: if not debug: self._engine.clear_tempdir('evaluation')