from config import VOCAB_HOME
from acqdivReader import AcqdivReader
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("--language", dest="language", type=str)
import random
args = parser.parse_args()
print(args)
acqdivCorpusReader = AcqdivReader(args.language)
vocabulary = set()
iterator = acqdivCorpusReader.iterator()
for utterance in iterator:
utterance = utterance.split(" ")
for word in utterance:
vocabulary.add(word)
with open(VOCAB_HOME + args.language + '-vocab.txt', "w") as outFile:
for word in vocabulary:
print(word, file=outFile)
parser = argparse.ArgumentParser()
parser.add_argument("--language", dest="language", type=str)
import random
args = parser.parse_args()
print(args)
from acqdivReader import AcqdivReader, AcqdivReaderPartition
acqdivCorpusReader = AcqdivReader(args.language)
import syllabificationJapanese
syllables = {}
for chunk in acqdivCorpusReader.iterator():
# print(len(chunk))
# tokenize chunk into valid syllables
words = chunk.split(" ")
for word in words:
if word == "\n":
continue
if len(word) == 0:
continue
if word == "n":
continue
if word == "???":
continue
syllabification = syllabificationJapanese.syllabify(word)
if syllabification is None: