def addItem(self, key):
"""Add pending item to DB"""
key = unicode(key, 'utf-8')
# Check, if key already exists
if len(Key.objects(value=key)) == 1:
return None
# Detect language
detected = set(Language().detect(key))
supported = [
lang for lang in languages
if detected.intersection(languages.get(lang))
]
# Supported language detected
if supported:
# NB: should deinflect to baseform (not so simple, actually!)
item = Key(value=key, lang=supported.pop())
if(item.lang == 'Japanese'):
# Set tags
item.tags = ['testing']
# Detect part of speech
# NB: should probably do this in POST-PROCESSING
item.pos = MeCab().partOfSpeech(item.value)
#item.pos, base = MeCab().partOfSpeech(item.value)
#if base != item.value:
#item.value = base
# TODO: get type (somehow, based on pos)
# TODO: if noun & 1 symbol == kanji, if two and more = word...
if len(item.value) == 1:
item.category = 'kanji'
elif item.pos == '':
item.category = 'compound'
else:
item.category = 'word'
# Unprocesed item
item.status = 'new'
# Save item
item.save()
return item
# Unsupported language
else:
return None
def process(self, category='kanji', limit=100):
"""Process all new & unprocessed kanji keys"""
wn = Wordnet()
mc = MeCab()
ji = Jisho()
# 0. Find unprocessed kanji key
try:
for key in Key.objects(
category=category, status='new'
).timeout(False).limit(limit):
print 'Processing ', key.value
# 0a. Get reading for kanji itself
key_reading = mc.reading(key.value)
key_gloss = Gloss()
key_gloss.readings.update({'default': key_reading})
key_gloss.save()
# 0b. Initialize corresponding Fact
key_fact = Fact(key=key, gloss=key_gloss)
# 1. Get usages from WordNet
words = wn.complete(key.value)
if words:
for word in words[:7]:
# 2. Check, if reading is found
reading = mc.reading(word)
if(not reading):
continue
# 3. Check, if definition is found
definitions = wn.lookup(word)
if(not definitions):
continue
# 4. Create new Key and corresponding Fact entities
try:
# Check if such item already exists
existing_key = Key.objects.get(value=word)
fact = existing_key.fact
except (DoesNotExist, MultipleObjectsReturned):
# 5a. Create Gloss entity for most common definitions
gloss = Gloss()
# No more than 2-4 definitions!
for definition in definitions[:3]:
gloss.translations.append(definition['gloss'])
gloss.readings.update({'default': reading})
gloss.save()
# 5b. Create corresponding key & fact
new_key = Key(
value=word,
category='word',
tags=['minor']
).save()
fact = Fact(key=new_key, gloss=gloss).save()
new_key.fact = fact
new_key.status = 'processed'
new_key.save()
# TODO: add synonyms based on 'words'?
# TODO: parse components?
# TODO: find advanced examples?
#6. Link fact to key-fact as usages
key_fact.usages.append(fact)
# 1a. If still no usages found (or not enough)
if len(key_fact.usages) < 2:
words = ji.define(key.value, 7)
for word, info in words:
# 4. Create new Key and corresponding Fact entities
try:
# Check if such item already exists
existing_key = Key.objects.get(value=word)
fact = existing_key.fact
except (DoesNotExist, MultipleObjectsReturned):
# 5a. Create Gloss entity for most common definitions
gloss = Gloss()
gloss.translations.append(info['meaning'])
gloss.readings.update({'default': info['kana']})
gloss.save()
# 5b. Create corresponding key & fact
new_key = Key(
value=word,
category='word',
tags=['minor']
).save()
fact = Fact(key=new_key, gloss=gloss).save()
new_key.fact = fact
new_key.status = 'processed'
new_key.save()
#6. Link fact to key-fact as usages
key_fact.usages.append(fact)
#7. Save key fact and corresponding key (bi-directional link)
key_fact.save()
key.fact = key_fact
if len(key_fact.usages) > 0:
# todo: if still nothing found -> lookup in names
# dictionary (jisho)
key.status = 'processed'
key.save()
print 'Total usages: ', len(key.usages())
print '----------------'
except OperationFailure as e:
print 'There was an error querying mongo db: %s' % e