Esempio n. 1
0
def tokenize(x, t):
    if t in TOKC:
        from jieba import posseg
        toks = posseg.cut(x)
        if t == POSC:
            return u'\u3000'.join([('%s [%s]' % (f.word, f.flag))
                                   for f in toks])
        elif t == SPACEC:
            return u'\u3000'.join([('%s' % (f.word)) for f in toks])
        else:
            return lexDens(toks, t)
    elif t in TOKJ:
        from rakutenma import RakutenMA
        rma = RakutenMA()
        rma = RakutenMA(phi=1024, c=0.007812)
        tD, tF = os.path.split(__file__)
        jSon = os.path.join(tD, 'model_ja.min.json')
        rma.load(jSon)
        toks = rma.tokenize(x)
        if t == SPACEJ:
            return u'\u3000'.join([i[0] for i in toks])
        elif t == POSJ:
            return u'\u3000'.join([('%s [%s]' % (i[0], i[1])) for i in toks])
        else:
            return lexDens(toks, t)
Esempio n. 2
0
    def __init__(self):
        super().__init__()

        # Initialize a RakutenMA instance with an empty model
        # the default ja feature set is set already
        self.rma = RakutenMA()

        # Initialize a RakutenMA instance with a pre-trained model
        self.rma = RakutenMA(
            phi=1024, c=0.007812
        )  # Specify hyperparameter for SCW (for demonstration purpose)

        # https://github.com/ikegami-yukino/rakutenma-python/tree/master/rakutenma/model
        self.rma.load(abspath(r'..\resource\model_ja.min.json'))
Esempio n. 3
0
 def tagWordsInSentences(self, studying, entry):
     '''Tags the part of speech for each word.'''
     jar_path = 'stanford-postagger-full/stanford-postagger.jar'
     if studying in self.english:
         words = parseWordsFromEntry(entry)
         tagged_words = tagWords(words)
         return tagged_words
     elif studying in self.japanese or self.korean or self.mandarin:
         #segmenter = TinySegmenter()
         #words = segmenter.tokenize(entry)
         rm = RakutenMA()
         tagged_words = rm.tokenize(entry)
         #mecab = Mecab()
         #tagged_words = mecab.pos(entry)
         return tagged_words
     else:
         if studying in self.spanish:
             model_path = 'stanford-postagger-full/models/spanish.tagger'
             words = parseWordsFromEntry(entry)
         elif studying in self.french:
             model_path = 'stanford-postagger-full/models/french.tagger'
             words = parseWordsFromEntry(entry)
         postagger = StanfordPOSTagger(model_path,
                                       jar_path,
                                       encoding='utf8')
         tagged_words = postagger.tag(words)
         return tagged_words
Esempio n. 4
0
def _get_tokenizer(lang):
    rma = None
    if lang == 'ja':
        rma = RakutenMA()
        rma.load('model_ja.json')
        rma.hash_func = rma.create_hash_func(15)
        tokenizer = _jap_tokenizer
        # tokenizer = _jap_character_tokenizer
    else:
        tokenizer = _eng_tokenizer
    return tokenizer, rma
Esempio n. 5
0
    def __init__(self, kv_filepath, model):
        self.rma = RakutenMA(json.loads(open(model).read()))
        self.rma.hash_func = RakutenMA.create_hash_func(self.rma, 15)


        self.ja_to_en = defaultdict(list)
        self.en_to_ja = defaultdict(list)

        for l in open(kv_filepath):
            [k, v] = l.strip().split(',')[:2]
            raw = unicode(k, 'utf-8')
#            lemma = self.rma.tokenize(raw)[0][0]
            self.ja_to_en[raw].append(v)
            self.en_to_ja[v].append(raw)
Esempio n. 6
0
def splitShuffle(expr, t):
    expr = stripHTML(expr).strip()
    if t == SGJ:       
        from rakutenma import RakutenMA    
        rma = RakutenMA(phi=1024, c=0.007812)
        tD, tF = os.path.split(__file__)
        jSon = os.path.join(tD, 'model_ja.min.json')     
        rma.load(jSon)           
        resultl = rma.tokenize(expr)  
        result = [r for r, s in resultl]
    elif t in SGC:    
        import jieba
        result = jieba.cut(expr, cut_all=False)  
    elif t == JSS:    
        result = expr.split(' ')
    elif t in WRAPL:
        result = list(expr)
    newResult, glosses = getResult(result, t)
    jn = u''
    full = jn.join(newResult)
    random.shuffle(newResult)     
    strResult = u''.join(newResult)     
    return strResult, full, glosses
Esempio n. 7
0
import os
import json
import pickle
from pathlib import Path
from transformers import BertTokenizer
from BertDataset import BertDataset
from tqdm import tqdm
import pandas as pd
import numpy as np
import unicodedata
import re
from rakutenma import RakutenMA

import sys

rma = RakutenMA()  # (default: phi = 2048, c = 0.003906)
tokenizer = BertTokenizer.from_pretrained('cl-tohoku/bert-base-japanese',
                                          do_lower_case=False)


def main(args):

    with open(args.config_path / 'config.json') as f:
        config = json.load(f)

    rma.load(args.config_path / "model_ja.min.json")
    rma.hash_func = rma.create_hash_func(15)

    print(f"config:{config}")

    # loading datasets from excel files
Esempio n. 8
0
 def __init__(self, model):
     print model
     self.rma = RakutenMA(json.loads(open(model).read()))
     self.rma.hash_func = RakutenMA.create_hash_func(self.rma, 15)
     return
Esempio n. 9
0
    del counter['DET']
    del counter['DETWH']
    counter['ET'] += 0
    counter['I'] += 0
    counter['NC'] += counter['N'] + counter['VINF']; del counter['N']; del counter['VINF']
    counter['NP'] += counter['NPP']; del counter['NPP']
    del counter['P']   # The Japanese tag set doesn't account for prepositions. We could manually look for them using a table like this: http://mylanguages.org/japanese_prepositions.php
    counter['PREF'] += 0
    counter['PRO'] += counter['PROREL'] + counter['PROWH']; del counter['PROREL']; del counter['PROWH']
    counter['V'] += counter['VIMP'] + counter['VPR'] + counter['VS']; del counter['VIMP']; del counter['VPR']; del counter['VS']
    counter['PUNC'] += counter['.$$.']; del counter['.$$.']

    return dict(counter)


rma = RakutenMA()
rma.load("model_ja.json")
def _analyze_ja(text):
    tags = rma.tokenize(text)
    counter = collections.Counter([x[1] for x in tags])  # see the same premise above in the French section
    subordinating_conjunctions = list(filter(lambda tup: tup[1] == 'C' and tup[0] in jsc, tags))

    return {  # we need to map the Japanese tagset to a subset of the French tagset, so that we can compare the two
        'ADJ':    counter['A-c'] + counter['A-dp'] + counter['J-c'] + counter['J-tari'] + counter['J-xs'] + counter['R'],
        'ADV':  counter['F'],
        'CC':   counter['C'] - len(subordinating_conjunctions),
        'CS':   len(subordinating_conjunctions),
        'ET':   counter['E'],
        'I':    counter['I-c'],
        'NC':   counter['N-n'] + counter['N-nc'],
        'NP':   counter['N-pn'],
Esempio n. 10
0
 def __init__(self):
     rma = RakutenMA()
     rma.load("model_ja.json")
     rma.hash_func = rma.create_hash_func(15)
     self.rma = rma 
# coding=utf8
from rakutenma import RakutenMA
import tinysegmenter
from nltk import *
import nltk
import re

#segmenter = tinysegmenter.TinySegmenter()
result = tinysegmenter.tokenize(
    "米中間選挙は6日に午後6時(日本時間7日午前8時)に一部の投票所が締め切られ、開票が始まった。米連邦議会の多数党がどちらになるかによって、ドナルド・トランプ米大統領の政策の行方が決まる。特に下院でどれだけ、民主党が共和党現職の議席を奪うかが注目されている。")
print('Segmenter: ')
print(result)

# Initialize a RakutenMA instance with an empty model
# the default ja feature set is set already
rma = RakutenMA()

# Let's analyze a sample sentence (from http://tatoeba.org/jpn/sentences/show/103809)
# With a disastrous result, since the model is empty!
print('Result')
print(rma.tokenize(result))
print('Original')
print(rma.tokenize("米中間選挙は6日に午後6時(日本時間7日午前8時)に一部の投票所が締め切られ、開票が始まった。"))
print('------------------')
# print(rma.tokenize("子どものみなさん、ゆるしてください。ぼくはこの本をひとりのおとなのひとにささげます。でもちゃんとしたわけがあるのです。"))
# print(rma.tokenizetwo("彼は新しい仕事できっと成功するだろう。"))
# print(rma.tokenize("彼は新しい仕事できっと成功するだろう。"))

# Feed the model with ten sample sentences from tatoeba.com
# "tatoeba.json" is available at https://github.com/rakuten-nlp/rakutenma