def __init__(self, lang='nl', tokens_glue=None, tokens_aligner=fTokensAligner( fTokensAligner.fRegexpTokenFinder())): assert tokens_aligner Tokenizer.__init__(self, lang) self.text_parse_re = re.compile('([^/\s]+/[A-Z0]+)', re.U) self.tokens_aligner = tokens_aligner self.glue = tokens_glue
def __init__(self, lang = 'nl'): Tokenizer.__init__(self, lang) self.memory, self.tags_map = "-mx512M", {} self.class_loader = self.ner_instance = None # test wheher tokenizer has been started under Jython try: from java.lang import Class self.class_loader = Class.forName("gravity.common.CustomClassLoader")(self.java_classpath()) except: pass
def __init__(self, lang="nl"): Tokenizer.__init__(self, lang)
def __init__(self, lang='nl'): Tokenizer.__init__(self, lang)
def __init__(self, lang='nl', tokens_glue = None, tokens_aligner =fTokensAligner(fTokensAligner.fRegexpTokenFinder())): assert tokens_aligner Tokenizer.__init__(self, lang) self.text_parse_re = re.compile('([^/\s]+/[A-Z0]+)', re.U) self.tokens_aligner = tokens_aligner self.glue = tokens_glue