def train_morfessor(target, source, env): """Train a Morfessor model using a word list as input. This builder is largely based on the code in the new Python version of Morfessor. Note that it prevents splitting that would create a morph composed just of non-acoustic graphemes. Sources: word list file Targets: segmented word list file, morfessor model file """ parser = get_default_argparser() args = parser.parse_args([]) dampfunc = lambda x : x if len(env.get("NON_ACOUSTIC_GRAPHEMES", [])) > 0: rx_str = "((\\S(%(ALT)s))|(^(%(ALT)s)\\S))" % {"ALT" : "|".join([unichr(int(x, base=16)) for x in env.get("NON_ACOUSTIC_GRAPHEMES")])} else: rx_str = None model = BaselineModel(forcesplit_list=env.get("FORCE_SPLIT", []), corpusweight=1.0, use_skips=False, nosplit_re=rx_str) io = MorfessorIO(encoding=args.encoding, compound_separator=args.cseparator, atom_separator=args.separator) words = {} with meta_open(source[0].rstr()) as ifd: for line in ifd: toks = line.strip().split() for word in toks[0].split("-"): if len(toks) == 1: words[word] = 1 elif len(toks) == 2: words[word] = words.get(word, 0) + int(toks[1]) else: return "malformed vocabulary line: %s" % (line.strip()) words = {w : c for w, c in words.iteritems() if not re.match(env.get("NON_WORD_PATTERN", "^$"), w)} model.load_data([(c, w, (w)) for w, c in words.iteritems()], args.freqthreshold, dampfunc, args.splitprob) algparams = () develannots = None e, c = model.train_batch(args.algorithm, algparams, develannots, args.finish_threshold, args.maxepochs) with meta_open(target[0].rstr(), "w") as ofd: for n, morphs in model.get_segmentations(): if len(morphs) >= 2: morphs = ["%s+" % morphs[0]] + ["+%s+" % t for t in morphs[1:-1]] + ["+%s" % morphs[-1]] ofd.write(" ".join(morphs) + "\n") io.write_binary_model_file(target[1].rstr(), model) return None
def read_any_model(self, file_name): """Read a file that is either a binary model or a Morfessor 1.0 style model segmentation. This method can not be used on standard input as data might need to be read multiple times""" try: model = self.read_binary_model_file(file_name) _logger.info("%s was read as a binary model" % file_name) return model except BaseException: pass from morfessor import BaselineModel model = BaselineModel() model.load_segmentations(self.read_segmentation_file(file_name)) _logger.info("%s was read as a segmentation" % file_name) return model