def train_morfessor(target, source, env):
"""Train a Morfessor model using a word list as input.
This builder is largely based on the code in the new Python version of Morfessor.
Note that it prevents splitting that would create a morph composed just of
non-acoustic graphemes.
Sources: word list file
Targets: segmented word list file, morfessor model file
"""
parser = get_default_argparser()
args = parser.parse_args([])
dampfunc = lambda x : x
if len(env.get("NON_ACOUSTIC_GRAPHEMES", [])) > 0:
rx_str = "((\\S(%(ALT)s))|(^(%(ALT)s)\\S))" % {"ALT" : "|".join([unichr(int(x, base=16)) for x in env.get("NON_ACOUSTIC_GRAPHEMES")])}
else:
rx_str = None
model = BaselineModel(forcesplit_list=env.get("FORCE_SPLIT", []),
corpusweight=1.0,
use_skips=False,
nosplit_re=rx_str)
io = MorfessorIO(encoding=args.encoding,
compound_separator=args.cseparator,
atom_separator=args.separator)
words = {}
with meta_open(source[0].rstr()) as ifd:
for line in ifd:
toks = line.strip().split()
for word in toks[0].split("-"):
if len(toks) == 1:
words[word] = 1
elif len(toks) == 2:
words[word] = words.get(word, 0) + int(toks[1])
else:
return "malformed vocabulary line: %s" % (line.strip())
words = {w : c for w, c in words.iteritems() if not re.match(env.get("NON_WORD_PATTERN", "^$"), w)}
model.load_data([(c, w, (w)) for w, c in words.iteritems()], args.freqthreshold, dampfunc, args.splitprob)
algparams = ()
develannots = None
e, c = model.train_batch(args.algorithm, algparams, develannots,
args.finish_threshold, args.maxepochs)
with meta_open(target[0].rstr(), "w") as ofd:
for n, morphs in model.get_segmentations():
if len(morphs) >= 2:
morphs = ["%s+" % morphs[0]] + ["+%s+" % t for t in morphs[1:-1]] + ["+%s" % morphs[-1]]
ofd.write(" ".join(morphs) + "\n")
io.write_binary_model_file(target[1].rstr(), model)
return None
def read_any_model(self, file_name):
"""Read a file that is either a binary model or a Morfessor 1.0 style
model segmentation. This method can not be used on standard input as
data might need to be read multiple times"""
try:
model = self.read_binary_model_file(file_name)
_logger.info("%s was read as a binary model" % file_name)
return model
except BaseException:
pass
from morfessor import BaselineModel
model = BaselineModel()
model.load_segmentations(self.read_segmentation_file(file_name))
_logger.info("%s was read as a segmentation" % file_name)
return model