def _analyze(db, fin, fout, backoff, cache):
if cache:
analyzer = Analyzer(db, backoff, cache_size=1024)
else:
analyzer = Analyzer(db, backoff)
line = force_unicode(fin.readline())
while line:
if len(line) == 0:
line = force_unicode(fin.readline())
continue
line = line.strip()
tokens = _tokenize(line)
for token in tokens:
analyses = analyzer.analyze(token)
serialized = _serialize_analyses(fout, token, analyses, db.order)
if six.PY3:
fout.write(serialized)
else:
fout.write(force_encoding(serialized))
fout.write('\n\n')
line = force_unicode(fin.readline())
def clean(self,
ar_raw_text: str,
normalize_teh_marbuta: bool = False,
normalize_alef: bool = True,
is_punct: bool = False,
is_de_diacritize: bool = True,
is_to_lower: bool = True) -> str:
clean_text = ar_raw_text.strip("\n").strip()
if is_to_lower:
clean_text = clean_text.lower()
clean_text = force_unicode(clean_text)
if six.PY3:
clean_text = self.clean_mapper.map_string(clean_text)
else:
clean_text = force_encoding(clean_text)
clean_text = self.clean_mapper.map_string(clean_text)
clean_text = self._get_sec_cleaner(
is_punct=is_punct).clean_text(clean_text)
clean_text = self._normalize(
arb_text=clean_text,
normalize_teh_marbuta=normalize_teh_marbuta,
normalize_alef=normalize_alef)
if is_de_diacritize:
clean_text = self._de_diacritize(arb_text=clean_text)
for i in range(5):
clean_text = clean_text.replace(" ", " ")
return clean_text.strip()
def _arclean(mapper, fin, fout):
for line in fin:
line = force_unicode(line)
if six.PY3:
fout.write(mapper.map_string(line))
else:
fout.write(force_encoding(mapper.map_string(line)))
fout.flush()
def _analyze(db, fin, fout, backoff, cache):
analyzer = CalimaStarAnalyzer(db, backoff)
memoize_table = {} if cache else None
line = force_unicode(fin.readline())
while line:
if len(line) == 0:
line = force_unicode(fin.readline())
continue
line = line.strip()
tokens = _tokenize(line)
for token in tokens:
if cache and token in memoize_table:
if six.PY3:
fout.write(memoize_table[token])
else:
fout.write(force_encoding(memoize_table[token]))
fout.write('\n\n')
else:
analyses = analyzer.analyze(token)
serialized = _serialize_analyses(fout, token, analyses,
db.order)
if cache:
memoize_table[token] = serialized
if six.PY3:
fout.write(serialized)
else:
fout.write(force_encoding(serialized))
fout.write('\n\n')
line = force_unicode(fin.readline())
def _reinflect(db, fin, fout):
reinflector = Reinflector(db)
line = force_unicode(fin.readline())
line_num = 1
while line:
line = line.strip()
if len(line) == 0:
line = force_unicode(fin.readline())
line_num += 1
continue
parsed = _parse_reinflector_line(line)
if parsed is None:
if fin is sys.stdin:
sys.stderr.write('Error: Invalid input line.\n')
else:
sys.stderr.write(
'Error: Invalid input line. [{}]\n'.format(line_num))
else:
word = parsed[0]
feats = parsed[1]
try:
analyses = reinflector.reinflect(word, feats)
serialized = _serialize_analyses(fout, word, analyses,
db.order)
if six.PY3:
fout.write(serialized)
else:
fout.write(force_encoding(serialized))
fout.write('\n\n')
except MorphologyError as error:
# This could be thrown by the analyzer, generator, or
# reinflector.
if fin is sys.stdin:
sys.stderr.write('Error: {}.\n'.format(error.msg))
else:
sys.stderr.write('Error: {}. [{}]\n'.format(
error.msg, line_num))
line = force_unicode(fin.readline())
line_num += 1
def _analyze(db, fin, fout, backoff, cache, num_disambig=None):
if cache:
analyzer = CalimaStarAnalyzer(db, backoff, cache_size=1024)
else:
analyzer = CalimaStarAnalyzer(db, backoff)
disambig = None
if num_disambig is not None:
disambig = MLEDisambiguator(analyzer)
line = force_unicode(fin.readline())
while line:
if len(line) == 0:
line = force_unicode(fin.readline())
continue
line = line.strip()
tokens = _tokenize(line)
for token in tokens:
analyses = analyzer.analyze(token)
if num_disambig is not None:
dambg = disambig.disambiguate([token], num_disambig)
analyses = [a.analysis for a in dambg[0].analyses]
else:
analyses = analyzer.analyze(token)
serialized = _serialize_analyses(fout, token, analyses, db.order)
if six.PY3:
fout.write(serialized)
else:
fout.write(force_encoding(serialized))
fout.write('\n\n')
line = force_unicode(fin.readline())
def main(): # pragma: no cover
try:
version = ('CAMeL Tools v{}'.format(__version__))
arguments = docopt(__doc__, version=version)
if arguments['--list']:
for scheme in _BUILTIN_SCHEMES:
print("{} {}".format(scheme[0].ljust(20), scheme[1]))
sys.exit(0)
if arguments['--scheme'] is not None:
if arguments['--scheme'] not in [s[0] for s in _BUILTIN_SCHEMES]:
sys.stderr.write('Error: {} is not a valid scheme.\n'
'Run `camel_transliterate -l` to see the list'
' of available schemes.'
'\n'.format(repr(arguments['--scheme'])))
sys.exit(1)
if arguments['--marker'] is None:
marker = '@@IGNORE@@'
else:
marker = arguments['--marker']
ignore_markers = arguments['--ignore-markers']
strip_markers = arguments['--strip-markers']
# Open files (or just use stdin and stdout)
fin, fout = _open_files(arguments['FILE'], arguments['--output'])
# Load the CharMapper and initialize a Transliterator with it
try:
mapper = CharMapper.builtin_mapper(arguments['--scheme'])
trans = Transliterator(mapper, marker)
except Exception: # pylint: disable=W0703
sys.stderr.write('Error: Could not load builtin scheme'
' {}.\n'.format(repr(arguments['--scheme'])))
sys.exit(1)
# Transliterate lines
try:
for line in fin:
line = force_unicode(line)
if six.PY3:
fout.write(
trans.transliterate(line, strip_markers,
ignore_markers))
else:
fout.write(
force_encoding(
trans.transliterate(line, strip_markers,
ignore_markers)))
fout.flush()
# If everything worked so far, this shouldn't happen
except Exception: # pylint: disable=W0703
sys.stderr.write('Error: An unkown error occured during '
'transliteration.\n')
sys.exit(1)
# Cleanup
if arguments['FILE'] is not None:
fin.close()
if arguments['--output'] is not None:
fout.close()
sys.exit(0)
except KeyboardInterrupt:
sys.stderr.write('Exiting...\n')
sys.exit(1)
except Exception:
sys.stderr.write('Error: An unknown error occurred.\n')
sys.exit(1)
def _generate(db, fin, fout, backoff):
generator = Generator(db)
reinflector = Reinflector(db) if backoff == 'REINFLECT' else None
line = force_unicode(fin.readline())
line_num = 1
while line:
line = line.strip()
if len(line) == 0:
line = force_unicode(fin.readline())
line_num += 1
continue
parsed = _parse_generator_line(line)
if parsed is None:
if fin is sys.stdin:
sys.stderr.write('Error: Invalid input line.\n')
else:
sys.stderr.write(
'Error: Invalid input line ({}).\n'.format(line_num))
else:
lemma = parsed[0]
feats = parsed[1]
# Make sure lemma and pos are specified first
if lemma is None:
if fin is sys.stdin:
sys.stderr.write('Error: Missing lex/lemma feature.\n')
else:
sys.stderr.write(
'Error: Missing lex/lemma feature. [{}].\n'.format(
line_num))
elif 'pos' not in feats:
if fin is sys.stdin:
sys.stderr.write('Error: Missing pos feature.\n')
else:
sys.stderr.write(
'Error: Missing pos feature. [{}]\n'.format(line_num))
else:
try:
analyses = generator.generate(lemma, feats)
if len(analyses) == 0 and backoff == 'REINFLECT':
word = _dediac(lemma)
analyses = reinflector.reinflect(word, feats)
serialized = _serialize_analyses(fout, lemma, analyses,
db.order, True)
if six.PY3:
fout.write(serialized)
else:
fout.write(force_encoding(serialized))
fout.write('\n\n')
except GeneratorError as error:
if fin is sys.stdin:
sys.stderr.write('Error: {}.\n'.format(error.msg))
else:
sys.stderr.write('Error: {}. [{}]\n'.format(
error.msg, line_num))
line = force_encoding(fin.readline())
line_num += 1