parser.add_argument('--tokenized1', required=True,
help='Path to the file with the tokenized text of the documents crawled in LANG1')
parser.add_argument('--text2', help='Path to the file with the plain text of the documents in LANG2', required=True)
parser.add_argument('--tokenized2', required=True,
help='Path to the file with the tokenized text of the documents crawled in LANG2')
parser.add_argument('--column1', help='Column that contains the first document of the document pairs',
default=0, type=int)
parser.add_argument('--column2', help='Column that contains the second document of the document pairs',
default=1, type=int)
args = parser.parse_args()
lang2_docs = set()
lang2_read_docs = {}
with open_xz_or_gzip_or_plain(args.indices, 'rt') as reader, \
open_xz_or_gzip_or_plain(args.tokenized1) as tok_reader1, \
open_xz_or_gzip_or_plain(args.tokenized2) as tok_reader2, \
open_xz_or_gzip_or_plain(args.text1) as text_reader1, \
open_xz_or_gzip_or_plain(args.text2) as text_reader2:
for line in reader:
fields = line.split('\t')
lang2_docs.add(int(fields[args.column2]))
reader.seek(0)
doc1_current_line = 1
doc2_current_line = 1
doc2_last_written = 0
lang = os.fsdecode(langfolder)
if not os.path.isdir(options.folder + "/" + lang) or len(lang) > 2:
continue
fullname = os.path.join(options.folder, lang + "/plain_text.xz")
if os.path.isfile(fullname) and (not langs or lang in langs):
if lang not in lang_files:
lang_files[lang] = lzma.open(
os.path.join(options.folder, lang + "/plain_tokenized.xz"),
"wb")
senttok_command = get_lang_or_default(options.splitters, lang)
senttok = None
if senttok_command:
senttok = ToolWrapper(senttok_command.split())
wordtok_command = get_lang_or_default(options.tokenizers, lang)
wordtok = None
if wordtok_command:
wordtok = ToolWrapper(wordtok_command.split())
morphtok_command = get_lang_or_default(options.lemmatizers, lang)
morphtok = None
if morphtok_command:
morphtok = ToolWrapper(morphtok_command.split())
with open_xz_or_gzip_or_plain(fullname) as text_reader:
for line in text_reader:
encodedtext = line.strip()
tokenized = extract_encoded_text(encodedtext, senttok, wordtok,
morphtok)
lang_files[lang].write(
"{}\n".format(tokenized).encode("utf-8"))
for lang in lang_files:
lang_files[lang].close()
oparser.add_argument('--morph-analyser',
dest='lemmatizer',
help="Morphological analyser command")
oparser.add_argument(
'--sentences-output',
default="plain_sentences.xz",
dest='sent_output',
help="Path of the output file that will contain sentence splitted text")
oparser.add_argument(
'--tokenized-output',
default="plain_tokenized.xz",
dest='tok_output',
help=
"Path of the output file that will contain sentence splitted and tokenized text"
)
options = oparser.parse_args()
with open_xz_or_gzip_or_plain(options.text) as reader, lzma.open(
options.sent_output,
"w") as sent_writer, lzma.open(options.tok_output, "w") as tok_writer:
for line in reader:
encoded_text = line.strip()
sentences, tokenized = extract_encoded_text(encoded_text,
options.splitter,
options.tokenizer,
options.lemmatizer)
if sentences and tokenized:
sent_writer.write(sentences + b"\n")
tok_writer.write(tokenized + b"\n")
dest="no_delete_seg",
action='store_true')
oparser.add_argument("-f",
"--text-file-deduped",
help="Filename to write the deduped input file",
dest="text_file_deduped")
oparser.add_argument(
"--dedup",
dest="dedup",
help=
"Dedup entries and group urls using given columns. Like 'bifixerhash', 'seg1,seg2' , 'checksum1,checksum2'"
)
options = oparser.parse_args()
with open_xz_or_gzip_or_plain(options.clean_alignments, 'rt') if options.clean_alignments else sys.stdin as reader,\
open_xz_or_gzip_or_plain(options.text_file_deduped, 'wt') if options.text_file_deduped and options.dedup else dummy_open() as text_writer:
print("<?xml version=\"1.0\"?>")
print("<tmx version=\"1.4\">")
print(" <header")
print(" adminlang=\"" +
locale.setlocale(locale.LC_ALL, '').split(".")[0].split("_")[0] +
"\"")
print(" srclang=\"" + options.lang1 + "\"")
print(" o-tmf=\"PlainText\"")
print(" creationtool=\"bitextor\"")
print(" creationtoolversion=\"4.0\"")
print(" datatype=\"PlainText\"")
print(" segtype=\"sentence\"")
print(" creationdate=\"" + time.strftime("%Y%m%dT%H%M%S") + "\"")
oparser.add_argument(
"--lang2",
help="Two-characters-code for language 2 in the pair of languages",
dest="lang2",
required=True)
options = oparser.parse_args()
docnumber = 0
word_map = {}
punctuation = get_unicode_punct()
for file_path, lang in [(options.text1, options.lang1),
(options.text2, options.lang2)]:
with open_xz_or_gzip_or_plain(file_path) as text_reader:
for line in text_reader:
##################
# Parsing the text:
##################
tokenized_text = base64.b64decode(line.strip()).decode("utf-8")
sorted_uniq_wordlist = set(tokenized_text.split())
# Trimming non-aplphanumerics:
clean_sorted_uniq_wordlist = [
_f
for _f in [w.strip(punctuation) for w in sorted_uniq_wordlist]
if _f
]
required=True)
parser.add_argument('--columns2',
dest='lang2_column_filename',
nargs='+',
required=True)
args = parser.parse_args()
lang2_docs = set()
lang2_read_docs = {}
indices = list()
if not args.indices:
args.indices = '-'
with open_xz_or_gzip_or_plain(
args.indices) if args.indices != '-' else sys.stdin as reader:
for line in reader:
fields = line.strip().split('\t')
lang2_docs.add(int(fields[1]))
indices.append((int(fields[0]), int(fields[1])))
readers1 = [
open_xz_or_gzip(filename, 'rt')
for filename in args.lang1_column_filename
]
readers2 = [
open_xz_or_gzip(filename, 'rt')
for filename in args.lang2_column_filename
]
doc1_current_line = 1
help="Word tokeniser script for language 1",
dest="wordtokeniser1",
required=True)
oparser.add_argument("--wordtokeniser2",
help="Word tokeniser script for language 2",
dest="wordtokeniser2",
required=True)
options = oparser.parse_args()
docnumber = 0
word_map = {}
punctuation = get_unicode_punct()
with open_xz_or_gzip_or_plain(options.text) as text_reader:
with open_xz_or_gzip_or_plain(options.lang) as lang_reader:
for line in text_reader:
##################
# Parsing the text:
##################
text = base64.b64decode(line.strip()).decode("utf-8")
lang = next(lang_reader, None).strip()
proc = None
if len(
text.strip()
) != 0 and options.morphanal1 is not None and lang == options.lang1:
morphanalyser = ["/bin/bash", options.morphanal1]
spmorph = subprocess.Popen(morphanalyser,
proc_word = ExternalTextProcessor(word_tokeniser.split())
tokenized_text = proc_word.process(tokenized_filtered)
if morph_analyser:
proc_morph = ExternalTextProcessor(morph_analyser.split())
tokenized_text = proc_morph.process(tokenized_text)
b64sentences = base64.b64encode(tokenized_filtered.encode("utf-8"))
b64tokenized = base64.b64encode(tokenized_text.lower().encode("utf-8"))
return b64sentences, b64tokenized
oparser = argparse.ArgumentParser(
description="Tool that tokenizes (sentences, tokens and morphemes) plain text")
oparser.add_argument('--text', dest='text', help='Plain text file', required=True)
oparser.add_argument('--sentence-splitter', dest='splitter', required=True, help="Sentence splitter commands")
oparser.add_argument('--word-tokenizer', dest='tokenizer', required=True, help="Word tokenisation command")
oparser.add_argument('--morph-analyser', dest='lemmatizer', help="Morphological analyser command")
oparser.add_argument('--sentences-output', default="plain_sentences.xz", dest='sent_output', help="Path of the output file that will contain sentence splitted text")
oparser.add_argument('--tokenized-output', default="plain_tokenized.xz", dest='tok_output', help="Path of the output file that will contain sentence splitted and tokenized text")
options = oparser.parse_args()
with open_xz_or_gzip_or_plain(options.text) as reader, open_xz_or_gzip_or_plain(options.sent_output, "w") as sent_writer, open_xz_or_gzip_or_plain(options.tok_output, "w") as tok_writer:
for line in reader:
encoded_text = line.strip()
sentences, tokenized = extract_encoded_text(encoded_text, options.splitter, options.tokenizer, options.lemmatizer)
if sentences and tokenized:
sent_writer.write(sentences + b"\n")
tok_writer.write(tokenized + b"\n")
help="Write document alignments even if they are not backwards aligned ("
"option incompatible with 'iterations' option)",
dest="nonsymmetric",
action="store_true")
options = oparser.parse_args()
indices = {}
indicesProb = {}
documents = set(range(1, options.ndoc1 + options.ndoc2 + 1))
documentsFile2 = set()
file2_start_counter = options.ndoc1
if options.ridx2 is None:
# Reading the .ridx file with the preliminary alignment
with open_xz_or_gzip_or_plain(options.ridx1) if options.ridx1 else sys.stdin as reader, \
open_xz_or_gzip_or_plain(options.oridx, 'wt') if options.oridx else dummy_open() as oridx_writer:
for i in reader:
fields = i.split("\t")
if len(fields) >= 2:
if oridx_writer:
oridx_writer.write(i)
try:
indices[int(fields[0])] = int(
fields[1].strip().split(":")[0])
except:
pass
else:
with open_xz_or_gzip_or_plain(options.ridx1, 'rt') as reader1, \
open_xz_or_gzip_or_plain(options.ridx2, 'rt') as reader2, \
open_xz_or_gzip_or_plain(options.oridx, 'wt') if options.oridx else dummy_open() as oridx_writer:
splitter = options.splitter
splitter_func = None
# no sentence splitter command provided, use moses:
if not splitter:
splitter_func = split_moses
try:
if options.customnbp:
splitter = SentenceSplitter(
language=options.langcode,
non_breaking_prefix_file=options.customnbp)
else:
splitter = SentenceSplitter(language=options.langcode)
except SentenceSplitterException as e:
sys.stderr.write(str(e) + "\n")
splitter = SentenceSplitter(language='en')
# use custom sentence splitter via ExternalTextProcessor (inefficient):
else:
splitter_func = split_external
splitter = ExternalTextProcessor(os.path.expanduser(splitter).split())
with open_xz_or_gzip_or_plain(
options.text) if options.text != "-" else sys.stdin as reader:
for doc in reader:
content = base64.b64decode(doc.strip()).decode("utf-8").replace(
"\t", " ")
sentences = splitter_func(content, splitter, options.prune_type,
options.prune_threshold)
print(base64.b64encode(sentences.encode("utf-8")).decode("utf-8"))
reader1 = lzma.open(options.ridx1, "rt")
else:
reader1 = open(options.ridx1, "r")
if options.ridx2[-3:] == ".xz":
reader2 = lzma.open(options.ridx2, "rt")
else:
reader2 = open(options.ridx2, "r")
indices = {}
indicesProb = {}
documents = {}
documentsFile2 = set()
# Files containing base64 encoded text and url are read and stored in a document map
counter = 1
with open_xz_or_gzip_or_plain(options.text) as text_reader:
with open_xz_or_gzip_or_plain(options.url) as url_reader:
for text in text_reader:
url = next(url_reader, None).strip()
documents[counter] = (url, text.strip()) # URL parsed_text_base64
counter += 1
if not combine:
# Reading the .ridx file with the preliminary alignment
for i in reader:
fields = i.split("\t")
if len(fields) >= 2:
if options.oridx is not None:
options.oridx.write(i)
try:
indices[int(fields[0])] = int(fields[1].strip().split(":")[0])