def random(self, category='kanji'): """Get random item from collection""" try: return choice(Key.objects(category=category)) except: # No item found in this category return Key(value=u'')
def del_item(key): """Remove existing item""" results = {} for item in Key.objects(value=key): item.delete() results[key] = {'result': 'success'} return results
def random_with_usages(self, category='kanji'): """ Get random item, only if some usages provided NB: takes some time to compelete due to 'dumb' logic """ # Number of items of such category in DB tries = len(Key.objects(category=category)) while True: key = choice(Key.objects(category=category)) tries -= 1 if len(key.usages()) > 1 or tries <= 0: break # No item found in this category if tries <= 0: return Key(value=u'') else: return key
def list_all(): """ List all (AJAX) """ items = [] for i, item in enumerate(Key.objects()): items.append({ 'id': i, 'value': item.value, 'added': item.added.strftime("%Y-%m-%d %H:%M:%S"), 'pos': item.pos, 'status': item.status, 'category': item.category, 'lang': item.lang, }) # NB: bottle allows only dictionary serialization response.content_type = 'application/json' return dumps(items)
def addItem(self, key): """Add pending item to DB""" key = unicode(key, 'utf-8') # Check, if key already exists if len(Key.objects(value=key)) == 1: return None # Detect language detected = set(Language().detect(key)) supported = [ lang for lang in languages if detected.intersection(languages.get(lang)) ] # Supported language detected if supported: # NB: should deinflect to baseform (not so simple, actually!) item = Key(value=key, lang=supported.pop()) if(item.lang == 'Japanese'): # Set tags item.tags = ['testing'] # Detect part of speech # NB: should probably do this in POST-PROCESSING item.pos = MeCab().partOfSpeech(item.value) #item.pos, base = MeCab().partOfSpeech(item.value) #if base != item.value: #item.value = base # TODO: get type (somehow, based on pos) # TODO: if noun & 1 symbol == kanji, if two and more = word... if len(item.value) == 1: item.category = 'kanji' elif item.pos == '': item.category = 'compound' else: item.category = 'word' # Unprocesed item item.status = 'new' # Save item item.save() return item # Unsupported language else: return None
def get_item(self, value, category='kanji'): """Get item by its id and category""" return Key.objects(value=value, category='kanji').first()
def get(self, id, category='kanji'): """Get item by its id and category""" " TODO: use objects.get() instead of first" return Key.objects(id=id, category='kanji').first()
def export(self, category): """Return all keys in specified catgory""" return Key.objects(category=category)
def count(self, category='kanji', status='new'): """Count objects by category and status""" return len(Key.objects(category=category, status=status))
def process(self, category='kanji', limit=100): """Process all new & unprocessed kanji keys""" wn = Wordnet() mc = MeCab() ji = Jisho() # 0. Find unprocessed kanji key try: for key in Key.objects( category=category, status='new' ).timeout(False).limit(limit): print 'Processing ', key.value # 0a. Get reading for kanji itself key_reading = mc.reading(key.value) key_gloss = Gloss() key_gloss.readings.update({'default': key_reading}) key_gloss.save() # 0b. Initialize corresponding Fact key_fact = Fact(key=key, gloss=key_gloss) # 1. Get usages from WordNet words = wn.complete(key.value) if words: for word in words[:7]: # 2. Check, if reading is found reading = mc.reading(word) if(not reading): continue # 3. Check, if definition is found definitions = wn.lookup(word) if(not definitions): continue # 4. Create new Key and corresponding Fact entities try: # Check if such item already exists existing_key = Key.objects.get(value=word) fact = existing_key.fact except (DoesNotExist, MultipleObjectsReturned): # 5a. Create Gloss entity for most common definitions gloss = Gloss() # No more than 2-4 definitions! for definition in definitions[:3]: gloss.translations.append(definition['gloss']) gloss.readings.update({'default': reading}) gloss.save() # 5b. Create corresponding key & fact new_key = Key( value=word, category='word', tags=['minor'] ).save() fact = Fact(key=new_key, gloss=gloss).save() new_key.fact = fact new_key.status = 'processed' new_key.save() # TODO: add synonyms based on 'words'? # TODO: parse components? # TODO: find advanced examples? #6. Link fact to key-fact as usages key_fact.usages.append(fact) # 1a. If still no usages found (or not enough) if len(key_fact.usages) < 2: words = ji.define(key.value, 7) for word, info in words: # 4. Create new Key and corresponding Fact entities try: # Check if such item already exists existing_key = Key.objects.get(value=word) fact = existing_key.fact except (DoesNotExist, MultipleObjectsReturned): # 5a. Create Gloss entity for most common definitions gloss = Gloss() gloss.translations.append(info['meaning']) gloss.readings.update({'default': info['kana']}) gloss.save() # 5b. Create corresponding key & fact new_key = Key( value=word, category='word', tags=['minor'] ).save() fact = Fact(key=new_key, gloss=gloss).save() new_key.fact = fact new_key.status = 'processed' new_key.save() #6. Link fact to key-fact as usages key_fact.usages.append(fact) #7. Save key fact and corresponding key (bi-directional link) key_fact.save() key.fact = key_fact if len(key_fact.usages) > 0: # todo: if still nothing found -> lookup in names # dictionary (jisho) key.status = 'processed' key.save() print 'Total usages: ', len(key.usages()) print '----------------' except OperationFailure as e: print 'There was an error querying mongo db: %s' % e
def get_item(key): """ Get existing item """ return { 'found': [item.value for item in Key.objects(value=unicode(key, 'utf-8'))] }