def tokenize(x, t):
if t in TOKC:
from jieba import posseg
toks = posseg.cut(x)
if t == POSC:
return u'\u3000'.join([('%s [%s]' % (f.word, f.flag))
for f in toks])
elif t == SPACEC:
return u'\u3000'.join([('%s' % (f.word)) for f in toks])
else:
return lexDens(toks, t)
elif t in TOKJ:
from rakutenma import RakutenMA
rma = RakutenMA()
rma = RakutenMA(phi=1024, c=0.007812)
tD, tF = os.path.split(__file__)
jSon = os.path.join(tD, 'model_ja.min.json')
rma.load(jSon)
toks = rma.tokenize(x)
if t == SPACEJ:
return u'\u3000'.join([i[0] for i in toks])
elif t == POSJ:
return u'\u3000'.join([('%s [%s]' % (i[0], i[1])) for i in toks])
else:
return lexDens(toks, t)
def __init__(self):
super().__init__()
# Initialize a RakutenMA instance with an empty model
# the default ja feature set is set already
self.rma = RakutenMA()
# Initialize a RakutenMA instance with a pre-trained model
self.rma = RakutenMA(
phi=1024, c=0.007812
) # Specify hyperparameter for SCW (for demonstration purpose)
# https://github.com/ikegami-yukino/rakutenma-python/tree/master/rakutenma/model
self.rma.load(abspath(r'..\resource\model_ja.min.json'))
def tagWordsInSentences(self, studying, entry):
'''Tags the part of speech for each word.'''
jar_path = 'stanford-postagger-full/stanford-postagger.jar'
if studying in self.english:
words = parseWordsFromEntry(entry)
tagged_words = tagWords(words)
return tagged_words
elif studying in self.japanese or self.korean or self.mandarin:
#segmenter = TinySegmenter()
#words = segmenter.tokenize(entry)
rm = RakutenMA()
tagged_words = rm.tokenize(entry)
#mecab = Mecab()
#tagged_words = mecab.pos(entry)
return tagged_words
else:
if studying in self.spanish:
model_path = 'stanford-postagger-full/models/spanish.tagger'
words = parseWordsFromEntry(entry)
elif studying in self.french:
model_path = 'stanford-postagger-full/models/french.tagger'
words = parseWordsFromEntry(entry)
postagger = StanfordPOSTagger(model_path,
jar_path,
encoding='utf8')
tagged_words = postagger.tag(words)
return tagged_words
def _get_tokenizer(lang):
rma = None
if lang == 'ja':
rma = RakutenMA()
rma.load('model_ja.json')
rma.hash_func = rma.create_hash_func(15)
tokenizer = _jap_tokenizer
# tokenizer = _jap_character_tokenizer
else:
tokenizer = _eng_tokenizer
return tokenizer, rma
def __init__(self, kv_filepath, model):
self.rma = RakutenMA(json.loads(open(model).read()))
self.rma.hash_func = RakutenMA.create_hash_func(self.rma, 15)
self.ja_to_en = defaultdict(list)
self.en_to_ja = defaultdict(list)
for l in open(kv_filepath):
[k, v] = l.strip().split(',')[:2]
raw = unicode(k, 'utf-8')
# lemma = self.rma.tokenize(raw)[0][0]
self.ja_to_en[raw].append(v)
self.en_to_ja[v].append(raw)
def splitShuffle(expr, t):
expr = stripHTML(expr).strip()
if t == SGJ:
from rakutenma import RakutenMA
rma = RakutenMA(phi=1024, c=0.007812)
tD, tF = os.path.split(__file__)
jSon = os.path.join(tD, 'model_ja.min.json')
rma.load(jSon)
resultl = rma.tokenize(expr)
result = [r for r, s in resultl]
elif t in SGC:
import jieba
result = jieba.cut(expr, cut_all=False)
elif t == JSS:
result = expr.split(' ')
elif t in WRAPL:
result = list(expr)
newResult, glosses = getResult(result, t)
jn = u''
full = jn.join(newResult)
random.shuffle(newResult)
strResult = u''.join(newResult)
return strResult, full, glosses
import os
import json
import pickle
from pathlib import Path
from transformers import BertTokenizer
from BertDataset import BertDataset
from tqdm import tqdm
import pandas as pd
import numpy as np
import unicodedata
import re
from rakutenma import RakutenMA
import sys
rma = RakutenMA() # (default: phi = 2048, c = 0.003906)
tokenizer = BertTokenizer.from_pretrained('cl-tohoku/bert-base-japanese',
do_lower_case=False)
def main(args):
with open(args.config_path / 'config.json') as f:
config = json.load(f)
rma.load(args.config_path / "model_ja.min.json")
rma.hash_func = rma.create_hash_func(15)
print(f"config:{config}")
# loading datasets from excel files
def __init__(self, model):
print model
self.rma = RakutenMA(json.loads(open(model).read()))
self.rma.hash_func = RakutenMA.create_hash_func(self.rma, 15)
return
del counter['DET']
del counter['DETWH']
counter['ET'] += 0
counter['I'] += 0
counter['NC'] += counter['N'] + counter['VINF']; del counter['N']; del counter['VINF']
counter['NP'] += counter['NPP']; del counter['NPP']
del counter['P'] # The Japanese tag set doesn't account for prepositions. We could manually look for them using a table like this: http://mylanguages.org/japanese_prepositions.php
counter['PREF'] += 0
counter['PRO'] += counter['PROREL'] + counter['PROWH']; del counter['PROREL']; del counter['PROWH']
counter['V'] += counter['VIMP'] + counter['VPR'] + counter['VS']; del counter['VIMP']; del counter['VPR']; del counter['VS']
counter['PUNC'] += counter['.$$.']; del counter['.$$.']
return dict(counter)
rma = RakutenMA()
rma.load("model_ja.json")
def _analyze_ja(text):
tags = rma.tokenize(text)
counter = collections.Counter([x[1] for x in tags]) # see the same premise above in the French section
subordinating_conjunctions = list(filter(lambda tup: tup[1] == 'C' and tup[0] in jsc, tags))
return { # we need to map the Japanese tagset to a subset of the French tagset, so that we can compare the two
'ADJ': counter['A-c'] + counter['A-dp'] + counter['J-c'] + counter['J-tari'] + counter['J-xs'] + counter['R'],
'ADV': counter['F'],
'CC': counter['C'] - len(subordinating_conjunctions),
'CS': len(subordinating_conjunctions),
'ET': counter['E'],
'I': counter['I-c'],
'NC': counter['N-n'] + counter['N-nc'],
'NP': counter['N-pn'],
def __init__(self):
rma = RakutenMA()
rma.load("model_ja.json")
rma.hash_func = rma.create_hash_func(15)
self.rma = rma
# coding=utf8
from rakutenma import RakutenMA
import tinysegmenter
from nltk import *
import nltk
import re
#segmenter = tinysegmenter.TinySegmenter()
result = tinysegmenter.tokenize(
"米中間選挙は6日に午後6時（日本時間7日午前8時）に一部の投票所が締め切られ、開票が始まった。米連邦議会の多数党がどちらになるかによって、ドナルド・トランプ米大統領の政策の行方が決まる。特に下院でどれだけ、民主党が共和党現職の議席を奪うかが注目されている。")
print('Segmenter: ')
print(result)
# Initialize a RakutenMA instance with an empty model
# the default ja feature set is set already
rma = RakutenMA()
# Let's analyze a sample sentence (from http://tatoeba.org/jpn/sentences/show/103809)
# With a disastrous result, since the model is empty!
print('Result')
print(rma.tokenize(result))
print('Original')
print(rma.tokenize("米中間選挙は6日に午後6時（日本時間7日午前8時）に一部の投票所が締め切られ、開票が始まった。"))
print('------------------')
# print(rma.tokenize("子どものみなさん、ゆるしてください。ぼくはこの本をひとりのおとなのひとにささげます。でもちゃんとしたわけがあるのです。"))
# print(rma.tokenizetwo("彼は新しい仕事できっと成功するだろう。"))
# print(rma.tokenize("彼は新しい仕事できっと成功するだろう。"))
# Feed the model with ten sample sentences from tatoeba.com
# "tatoeba.json" is available at https://github.com/rakuten-nlp/rakutenma