def train_morfessor(target, source, env):
    """Train a Morfessor model using a word list as input.

    This builder is largely based on the code in the new Python version of Morfessor.
    Note that it prevents splitting that would create a morph composed just of
    non-acoustic graphemes.

    Sources: word list file
    Targets: segmented word list file, morfessor model file
    """
    parser = get_default_argparser()
    args = parser.parse_args([])
    dampfunc = lambda x : x
    if len(env.get("NON_ACOUSTIC_GRAPHEMES", [])) > 0:
        rx_str = "((\\S(%(ALT)s))|(^(%(ALT)s)\\S))" % {"ALT" : "|".join([unichr(int(x, base=16)) for x in env.get("NON_ACOUSTIC_GRAPHEMES")])}
    else:
        rx_str = None
    model = BaselineModel(forcesplit_list=env.get("FORCE_SPLIT", []),
                          corpusweight=1.0,
                          use_skips=False,
                          nosplit_re=rx_str)
    io = MorfessorIO(encoding=args.encoding,
                     compound_separator=args.cseparator,
                     atom_separator=args.separator)
    words = {}
    with meta_open(source[0].rstr()) as ifd:
        for line in ifd:
            toks = line.strip().split()
            for word in toks[0].split("-"):
                if len(toks) == 1:
                    words[word] = 1
                elif len(toks) == 2:                
                    words[word] = words.get(word, 0) + int(toks[1])
                else:
                    return "malformed vocabulary line: %s" % (line.strip())

    words = {w : c for w, c in words.iteritems() if not re.match(env.get("NON_WORD_PATTERN", "^$"), w)}
    model.load_data([(c, w, (w)) for w, c in words.iteritems()], args.freqthreshold, dampfunc, args.splitprob)
    algparams = ()
    develannots = None
    e, c = model.train_batch(args.algorithm, algparams, develannots,
                             args.finish_threshold, args.maxepochs)
    with meta_open(target[0].rstr(), "w") as ofd:
        for n, morphs in model.get_segmentations():
            if len(morphs) >= 2:
                morphs = ["%s+" % morphs[0]] + ["+%s+" % t for t in morphs[1:-1]] + ["+%s" % morphs[-1]]
            ofd.write(" ".join(morphs) + "\n")
    io.write_binary_model_file(target[1].rstr(), model)
    return None
Beispiel #2
0
    def read_any_model(self, file_name):
        """Read a file that is either a binary model or a Morfessor 1.0 style
        model segmentation. This method can not be used on standard input as
        data might need to be read multiple times"""
        try:
            model = self.read_binary_model_file(file_name)
            _logger.info("%s was read as a binary model" % file_name)
            return model
        except BaseException:
            pass

        from morfessor import BaselineModel
        model = BaselineModel()
        model.load_segmentations(self.read_segmentation_file(file_name))
        _logger.info("%s was read as a segmentation" % file_name)
        return model