def segmenteCorpus(corpus, storage): result = [] seg = Segmenter(storage) for page_id, pad in corpus: for line in pad.split("\n"): result.extend(["".join(w) for w in seg.segment(list(line))]) return result
def segmenteCorpus(corpus, storage): result = [] seg = Segmenter(storage) for page_id, pad in corpus: for line in pad.split("\n"): result.extend(["".join(w) for w in seg.segment(list(line))]) return result
def test_segmentation_2grams(storage): storage.add_sentence('je vous parle de hot dog'.split()) storage.add_sentence('j ador les hot dog'.split()) storage.add_sentence('hot dog ou pas'.split()) storage.add_sentence('hot dog ou sandwich'.split()) segmenter = Segmenter(storage) assert segmenter.segment('je deteste les hot dog'.split()) == [['je'], ['deteste'], ['les'], ['hot'], ['dog']]
def test_segmentation_basic(storage): storage.add_sentence('je vous parle de hot dog'.split()) storage.add_sentence('j ador les hot dog'.split()) storage.add_sentence('hot dog ou pas'.split()) storage.add_sentence('hot dog ou sandwich'.split()) segmenter = Segmenter(storage) assert segmenter.segment('je deteste les hot dog'.split()) == [['je'], ['deteste'], ['les'], ['hot', 'dog']] assert segmenter.segment('un chat noir et blanc'.split()) == [['un'], ['chat'], ['noir'], ['et'], ['blanc']]
def test_segmentation_2grams(storage): storage.add_sentence("je vous parle de hot dog".split()) storage.add_sentence("j ador les hot dog".split()) storage.add_sentence("hot dog ou pas".split()) storage.add_sentence("hot dog ou sandwich".split()) segmenter = Segmenter(storage) assert segmenter.segment("je deteste les hot dog".split()) == [ ["je"], ["deteste"], ["les"], ["hot"], ["dog"], ]
def benchmark(storage_class, create=True): m = storage_class(4) s = Segmenter(m, 3) if create: m.clear() corpus = reuters.raw() tokens = list(filter(lambda t: t.category == '', tokeniser_fr(corpus)))[:10000] if create: m.add_sentence(tokens) for i in range(1,5000,30): print(s.segment(tokens[i:i+30]))
def benchmark(storage_class, create=True): m = storage_class(4) s = Segmenter(m, 3) if create: m.clear() corpus = reuters.raw() tokens = list(filter(lambda t: t.category == "", tokeniser_fr(corpus)))[:10000] if create: m.add_sentence(tokens) for i in range(1, 5000, 30): print(s.segment(tokens[i:i + 30]))
def segment_with_preprocessing_pool(seg: Segmenter, sentences: List[str], bies: bool = True) -> List[str]: pool = Pool() tokenized_sentences = pool.map(tokenize_by_unicode_category, sentences) for sent in tokenized_sentences: tokens = [] for group in sent: try: if isCJK(group[0]): words = ["".join(w) for w in seg.segment(list(group))] for w in words: if bies: tokens.extend(add_bies(w)) else: tokens.append(w) else: if bies: tokens.extend(add_bies(group)) else: tokens.append(group) except: if bies: tokens.extend(add_bies(group)) else: tokens.append(group) yield " ".join(tokens) pool.terminate() pool.close()
def segment_file(storage, input_file: Path, output_file: Path): segmenter = Segmenter(storage) with open(input_file) as f: with open(output_file, "w") as out: for line in f: line = line[:-1] if line.strip() != "": out.write( chinese.segment_with_preprocessing(segmenter, line) + "\n")
def segment_batch(storage: Storage, batch: List[str]) -> List[str]: segmenter = Segmenter(storage) result = [] # pool = Pool() # for line in pool.map(lambda x: chinese.segment_with_preprocessing(segmenter,x), batch): for line in batch: line = line[:-1] if line.strip() != "": result.append(chinese.segment_with_preprocessing(segmenter, line)) return result
def addSegToBioUnsup(input: str, model: str, output: str, char_ind: int = 0, tag_ind: int = 1): from eleve import Segmenter from eleve.memory import CSVStorage lm = CSVStorage(model) s = Segmenter(lm) all = [] tags = [] sub = [] tag = [] with open(input, "r", encoding="utf-8") as fp: for line in fp: if line == "\n" or line == "": all.append(sub) tags.append(tag) sub = [] tag = [] else: line_split = line.replace("\n", "").split() c = line_split[char_ind] t = line_split[tag_ind] sub.append(c) tag.append(t) out = open(output, "w", encoding='utf-8') for i in range(len(all)): if len(all[i]) > 0: sentence = " ".join(all[i]) segmentation = s.segmentSentenceTIWBIES(sentence).split(" ") for j in range(len(segmentation)): line_out = segmentation[j] + " " + tags[i][j] + "\n" out.write(line_out) out.write('\n') out.close()
def test_segmentation_basic(storage): storage.add_sentence("je vous parle de hot dog".split()) storage.add_sentence("j ador les hot dog".split()) storage.add_sentence("hot dog ou pas".split()) storage.add_sentence("hot dog ou sandwich".split()) segmenter = Segmenter(storage) assert segmenter.segment("je deteste les hot dog".split()) == [ ["je"], ["deteste"], ["les"], ["hot", "dog"], ] assert segmenter.segment("un chat noir et blanc".split()) == [ ["un"], ["chat"], ["noir"], ["et"], ["blanc"], ]
def segment_with_preprocessing(seg: Segmenter, sent: str, bies: bool = True) -> str: tokens = [] for group in tokenize_by_unicode_category(sent): try: if isCJK(group[0]): words = ["".join(w) for w in seg.segment(list(group))] for w in words: if bies: tokens.extend(add_bies(w)) else: tokens.append(w) else: if bies: tokens.extend(add_bies(group)) else: tokens.append(group) except: if bies: tokens.extend(add_bies(group)) else: tokens.append(group) return " ".join(tokens)
from eleve import CMemoryStorage as Storage from eleve import Segmenter import sys corpus_file = sys.argv[1] with open(corpus_file) as f: storage = Storage(10) corpus = [] for line in f: tokens = list(line.strip().replace(" ", "")) storage.add_sentence(tokens) corpus.append(tokens) seg = Segmenter(storage) for tokens in corpus: result = seg.segment_nbest(tokens, 5) for ibest in result: print(" ".join(["".join(w) for w in ibest]))
print( '\t %s one-grams (non-unique) found in %s sentences'%\ ( '{:,}'.format(nMots), '{:,}'.format(nPosts)) ) print( '> ELeVE: Segment & count') dicoFr = lmdjxtools.getDicoFr() blacklist_afterEleve = lmdjxtools.blacklist_afterEleve() # Create table occurences cursor.execute('''DROP TABLE IF EXISTS occurences''') cursor.execute('''CREATE TABLE occurences (date text, ngram text, source text, postid integer)''') #postid? print('\t reset table DB.occurences') from eleve import Segmenter s = Segmenter(storage) rejected_ngrams = set() i = 0 cursor.execute('SELECT date, source, title, summary, rowid FROM posts') for line in cursor.fetchall(): date = line[0] source = line[1] postid = line[4] segmentedPhrase = [] for phrase in line[2:3]: formattedtext = lmdjxtools.format( phrase ) segmentedPhrase += s.segment( formattedtext.split(' ') )
print( '\t %s one-grams (non-unique) found in %s sentences'%\ ( '{:,}'.format(nMots), '{:,}'.format(nPosts)) ) print('> ELeVE: Segment & count') dicoFr = lmdjxtools.getDicoFr() blacklist_afterEleve = lmdjxtools.blacklist_afterEleve() # Create table occurences cursor.execute('''DROP TABLE IF EXISTS occurences''') cursor.execute('''CREATE TABLE occurences (date text, ngram text, source text, postid integer)''') #postid? print('\t reset table DB.occurences') from eleve import Segmenter s = Segmenter(storage) rejected_ngrams = set() i = 0 cursor.execute('SELECT date, source, title, summary, rowid FROM posts') for line in cursor.fetchall(): date = line[0] source = line[1] postid = line[4] segmentedPhrase = [] for phrase in line[2:3]: formattedtext = lmdjxtools.format(phrase) segmentedPhrase += s.segment(formattedtext.split(' ')) ngramAdded = set()