def search_path(rootpath, include_pairs=True, verbosity=1): lang_code = r'[a-z]{2,3}(?:_[A-Za-z0-9]+)?' type_re = { 'pair': re.compile(r'({0})-({0})\.mode'.format(lang_code)), 'analyzer': re.compile(r'(({0}(-{0})?)-(an)?mor(ph)?)\.mode'.format(lang_code)), 'generator': re.compile(r'(({0}(-{0})?)-gener[A-z]*)\.mode'.format(lang_code)), 'tagger': re.compile(r'(({0}(-{0})?)-tagger)\.mode'.format(lang_code)), 'spell': re.compile(r'(({0}(-{0})?)-spell)\.mode'.format(lang_code)), 'tokenise': re.compile(r'(({0}(-{0})?)-tokenise)\.mode'.format(lang_code)), } modes = { 'pair': [], 'analyzer': [], 'generator': [], 'tagger': [], 'spell': [], 'tokenise': [], } # type: Dict[str, List[Tuple[str, str, str]]] real_root = os.path.abspath(os.path.realpath(rootpath)) for dirpath, dirnames, files in os.walk(rootpath, followlinks=True): if is_loop(dirpath, rootpath, real_root): dirnames[:] = [] continue for filename in [f for f in files if f.endswith('.mode')]: for mtype, regex in type_re.items(): m = regex.match(filename) if m: if mtype != 'pair': modename = m.group(1) # e.g. en-es-anmorph langlist = [ to_alpha3_code(x) for x in m.group(2).split('-') ] lang_pair = '-'.join(langlist) # e.g. en-es dir_of_modes = os.path.dirname(dirpath) mode = (dir_of_modes, modename, lang_pair) modes[mtype].append(mode) elif include_pairs: lang_src = m.group(1) lang_trg = m.group(2) mode = (os.path.join(dirpath, filename), to_alpha3_code(lang_src), to_alpha3_code(lang_trg)) modes[mtype].append(mode) if verbosity > 1: _log_modes(modes) return modes
def search_path(rootpath, include_pairs=True, verbosity=1): lang_code = r'[a-z]{2,3}(?:_[A-Za-z]+)?' type_re = { 'pair': re.compile(r'({0})-({0})\.mode'.format(lang_code)), 'analyzer': re.compile(r'(({0}(-{0})?)-(an)?mor(ph)?)\.mode'.format(lang_code)), 'generator': re.compile(r'(({0}(-{0})?)-gener[A-z]*)\.mode'.format(lang_code)), 'tagger': re.compile(r'(({0}(-{0})?)-tagger)\.mode'.format(lang_code)), 'spell': re.compile(r'(({0}(-{0})?)-spell)\.mode'.format(lang_code)), 'tokenise': re.compile(r'(({0}(-{0})?)-tokenise)\.mode'.format(lang_code)), } modes = { 'pair': [], 'analyzer': [], 'generator': [], 'tagger': [], 'spell': [], 'tokenise': [], } # type: Dict[str, List[Tuple[str, str, str]]] real_root = os.path.abspath(os.path.realpath(rootpath)) for dirpath, dirnames, files in os.walk(rootpath, followlinks=True): if is_loop(dirpath, rootpath, real_root): dirnames[:] = [] continue for filename in [f for f in files if f.endswith('.mode')]: for mtype, regex in type_re.items(): m = regex.match(filename) if m: if mtype != 'pair': modename = m.group(1) # e.g. en-es-anmorph langlist = [to_alpha3_code(l) for l in m.group(2).split('-')] lang_pair = '-'.join(langlist) # e.g. en-es dir_of_modes = os.path.dirname(dirpath) mode = (dir_of_modes, modename, lang_pair) modes[mtype].append(mode) elif include_pairs: lang_src = m.group(1) lang_trg = m.group(2) mode = (os.path.join(dirpath, filename), to_alpha3_code(lang_src), to_alpha3_code(lang_trg)) modes[mtype].append(mode) if verbosity > 1: _log_modes(modes) return modes
def get(self): text = self.get_argument('q') if not text: return self.send_error(400, explanation='Missing q argument') if cld2: cld_results = cld2.detect(text) if cld_results[0]: possible_langs = filter(lambda x: x[1] != 'un', cld_results[2]) self.send_response({ to_alpha3_code(possible_lang[1]): possible_lang[2] for possible_lang in possible_langs }) else: self.send_response({'nob': 100 }) # TODO: Some more reasonable response else: try: coverages = yield gen.with_timeout( timedelta(seconds=self.timeout), get_coverages(text, self.analyzers, penalize=True), ) self.send_response(coverages) except gen.TimeoutError: self.send_error(408, explanation='Request timed out')
def get_pairs_or_error(self, langpairs, text_length): langs = [to_alpha3_code(lang) for lang in langpairs.split('|')] if len(langs) < 2: self.send_error( 400, explanation='Need at least two languages, use e.g. eng|spa') self.log_after_translation(self.log_before_translation(), text_length) return None if len(langs) == 2: if langs[0] == langs[1]: self.send_error( 400, explanation='Need at least two languages, use e.g. eng|spa' ) self.log_after_translation(self.log_before_translation(), text_length) return None return self.paths.get(langs[0], {}).get(langs[1]) for lang1, lang2 in self.pair_list(langs): if '{:s}-{:s}'.format(lang1, lang2) not in self.pairs: self.send_error( 400, explanation='Pair {:s}-{:s} is not installed'.format( lang1, lang2)) self.log_after_translation(self.log_before_translation(), text_length) return None return langs
def get(self): in_text = self.get_argument('q') + '*' in_mode = to_alpha3_code(self.get_argument('lang')) if '-' in in_mode: l1, l2 = map(to_alpha3_code, in_mode.split('-', 1)) in_mode = '%s-%s' % (l1, l2) in_mode = self.find_fallback_mode(in_mode, self.spellers) logging.info(in_text) logging.info(self.get_argument('lang')) logging.info(in_mode) logging.info(self.spellers) if in_mode in self.spellers: logging.info(self.spellers[in_mode]) [path, mode] = self.spellers[in_mode] logging.info(path) logging.info(mode) formatting = 'none' commands = [[ 'apertium', '-d', path, '-f', formatting, self.get_argument('lang') + '-tokenise' ]] result = yield translate_simple(in_text, commands) tokens = streamparser.parse(result) units = [] for token in tokens: if token.knownness == streamparser.known: units.append({ 'token': token.wordform, 'known': True, 'sugg': [] }) else: suggestion = [] commands = [[ 'apertium', '-d', path, '-f', formatting, mode ]] result = yield translate_simple(token.wordform, commands) found_sugg = False for line in result.splitlines(): if line.count('Corrections for'): found_sugg = True continue if found_sugg and '\t' in line: s, w = line.split('\t') suggestion.append((s, w)) units.append({ 'token': token.wordform, 'known': False, 'sugg': suggestion }) self.send_response(units) else: error_explanation = '{} on spellchecker mode: {}'.format( 'Error 404', 'Spelling mode for ' + in_mode + ' is not installed') self.send_error(404, explanation=error_explanation)
async def get(self): lang = to_alpha3_code(self.get_argument('lang')) modes = set(self.get_argument('modes').split(' ')) query = self.get_argument('q') if not modes <= { 'morph', 'biltrans', 'tagger', 'disambig', 'translate' }: self.send_error(400, explanation='Invalid mode argument') return def handle_output(output): """to_return = {} for mode in modes: to_return[mode] = outputs[mode] for mode in modes: to_return[mode] = {outputs[mode + '_inputs'][index]: output for (index, output) in enumerate(outputs[mode])} for mode in modes: to_return[mode] = [(outputs[mode + '_inputs'][index], output) for (index, output) in enumerate(outputs[mode])] for mode in modes: to_return[mode] = {'outputs': outputs[mode], 'inputs': outputs[mode + '_inputs']} self.send_response(to_return)""" if output is None: self.send_error(400, explanation='No output') return elif not output: self.send_error(408, explanation='Request timed out') return else: outputs, tagger_lexical_units, morph_lexical_units = output to_return = [] for (index, lexical_unit ) in enumerate(tagger_lexical_units if tagger_lexical_units else morph_lexical_units): unit_to_return = {} unit_to_return['input'] = strip_tags( lexical_unit.split('/')[0]) for mode in modes: unit_to_return[mode] = outputs[mode][index] to_return.append(unit_to_return) if self.get_argument('pos', default=None): requested_pos = int(self.get_argument('pos')) - 1 current_pos = 0 for unit in to_return: input = unit['input'] current_pos += len(input.split(' ')) if requested_pos < current_pos: self.send_response(unit) return else: self.send_response(to_return) output = await process_per_word(self.analyzers, self.taggers, lang, modes, query) handle_output(output)
def get(self): in_text = self.get_argument('q') in_mode = to_alpha3_code(self.get_argument('lang')) if in_mode in self.analyzers: [path, mode] = self.analyzers[in_mode] formatting = 'txt' commands = [['apertium', '-d', path, '-f', formatting, mode]] result = yield translate_simple(in_text, commands) self.send_response(self.postproc_text(in_text, result)) else: self.send_error(400, explanation='That mode is not installed')
def get(self): in_text = self.get_argument('q') in_mode = to_alpha3_code(self.get_argument('lang')) if in_mode in self.analyzers: [path, mode] = self.analyzers[in_mode] formatting = 'txt' commands = [['apertium', '-d', path, '-f', formatting, mode]] result = yield translate_simple(in_text, commands) self.send_response(self.postproc_text(in_text, result)) else: self.send_error(400, explanation='That mode is not installed')
def get(self): in_text = self.get_argument('q') in_mode = to_alpha3_code(self.get_argument('lang')) if in_mode in self.generators: [path, mode] = self.generators[in_mode] formatting = 'none' commands = [['apertium', '-d', path, '-f', formatting, mode]] lexical_units, to_generate = self.preproc_text(in_text) result = yield translate_simple(to_generate, commands) self.send_response(self.postproc_text(lexical_units, result)) else: self.send_error(400, explanation='That mode is not installed')
def get(self): mode = to_alpha3_code(self.get_argument('lang')) text = self.get_argument('q') if not text: self.send_error(400, explanation='Missing q argument') return if mode in self.analyzers: try: coverage = yield gen.with_timeout( timedelta(seconds=self.timeout), get_coverage(text, self.analyzers[mode][0], self.analyzers[mode][1]), ) self.send_response([coverage]) except gen.TimeoutError: self.send_error(408, explanation='Request timed out') else: self.send_error(400, explanation='That mode is not installed')
def get(self): text = self.get_argument('q') if not text: return self.send_error(400, explanation='Missing q argument') if cld2: cld_results = cld2.detect(text) if cld_results[0]: possible_langs = filter(lambda x: x[1] != 'un', cld_results[2]) self.send_response({to_alpha3_code(possible_lang[1]): possible_lang[2] for possible_lang in possible_langs}) else: self.send_response({'nob': 100}) # TODO: Some more reasonable response else: try: coverages = yield gen.with_timeout( timedelta(seconds=self.timeout), get_coverages(text, self.analyzers, penalize=True), ) self.send_response(coverages) except gen.TimeoutError: self.send_error(408, explanation='Request timed out')
def get(self): lang = to_alpha3_code(self.get_argument('lang')) modes = set(self.get_argument('modes').split(' ')) query = self.get_argument('q') if not modes <= {'morph', 'biltrans', 'tagger', 'disambig', 'translate'}: self.send_error(400, explanation='Invalid mode argument') return def handle_output(output): """to_return = {} for mode in modes: to_return[mode] = outputs[mode] for mode in modes: to_return[mode] = {outputs[mode + '_inputs'][index]: output for (index, output) in enumerate(outputs[mode])} for mode in modes: to_return[mode] = [(outputs[mode + '_inputs'][index], output) for (index, output) in enumerate(outputs[mode])] for mode in modes: to_return[mode] = {'outputs': outputs[mode], 'inputs': outputs[mode + '_inputs']} self.send_response(to_return)""" if output is None: self.send_error(400, explanation='No output') return elif not output: self.send_error(408, explanation='Request timed out') return else: outputs, tagger_lexical_units, morph_lexical_units = output to_return = [] for (index, lexical_unit) in enumerate(tagger_lexical_units if tagger_lexical_units else morph_lexical_units): unit_to_return = {} unit_to_return['input'] = strip_tags(lexical_unit.split('/')[0]) for mode in modes: unit_to_return[mode] = outputs[mode][index] to_return.append(unit_to_return) if self.get_argument('pos', default=None): requested_pos = int(self.get_argument('pos')) - 1 current_pos = 0 for unit in to_return: input = unit['input'] current_pos += len(input.split(' ')) if requested_pos < current_pos: self.send_response(unit) return else: self.send_response(to_return) pool = Pool(processes=1) result = pool.apply_async(process_per_word, (self.analyzers, self.taggers, lang, modes, query)) pool.close() @run_async_thread def worker(callback): try: callback(result.get(timeout=self.timeout)) except TimeoutError: pool.terminate() callback(None) output = yield gen.Task(worker) handle_output(output)
def get(self): lang = to_alpha3_code(self.get_argument('lang')) modes = set(self.get_argument('modes').split(' ')) query = self.get_argument('q') if not modes <= { 'morph', 'biltrans', 'tagger', 'disambig', 'translate' }: self.send_error(400, explanation='Invalid mode argument') return def handle_output(output): """to_return = {} for mode in modes: to_return[mode] = outputs[mode] for mode in modes: to_return[mode] = {outputs[mode + '_inputs'][index]: output for (index, output) in enumerate(outputs[mode])} for mode in modes: to_return[mode] = [(outputs[mode + '_inputs'][index], output) for (index, output) in enumerate(outputs[mode])] for mode in modes: to_return[mode] = {'outputs': outputs[mode], 'inputs': outputs[mode + '_inputs']} self.send_response(to_return)""" if output is None: self.send_error(400, explanation='No output') return elif not output: self.send_error(408, explanation='Request timed out') return else: outputs, tagger_lexical_units, morph_lexical_units = output to_return = [] for (index, lexical_unit ) in enumerate(tagger_lexical_units if tagger_lexical_units else morph_lexical_units): unit_to_return = {} unit_to_return['input'] = strip_tags( lexical_unit.split('/')[0]) for mode in modes: unit_to_return[mode] = outputs[mode][index] to_return.append(unit_to_return) if self.get_argument('pos', default=None): requested_pos = int(self.get_argument('pos')) - 1 current_pos = 0 for unit in to_return: input = unit['input'] current_pos += len(input.split(' ')) if requested_pos < current_pos: self.send_response(unit) return else: self.send_response(to_return) pool = Pool(processes=1) result = pool.apply_async( process_per_word, (self.analyzers, self.taggers, lang, modes, query)) pool.close() @run_async_thread def worker(callback): try: callback(result.get(timeout=self.timeout)) except TimeoutError: pool.terminate() callback(None) output = yield gen.Task(worker) handle_output(output)