def post(self, request, *args, **kwargs): logging.info('Get a message!') if request.method == 'POST': userText = request.POST['text'] addWord = request.POST['word'] addReading = request.POST['reading'] addDefinition = request.POST['definition'] addSentence = request.POST['sentence'] unknownWordsOnly = request.POST['unknownWordsOnly'] logger = logging.getLogger('console') logger.info('POST settings: userTextLen:%s, word:%s, reading:%s, ' 'definition:%s, sentence:%s, unknownWordsOnly:%s', len(userText), addWord, addReading, addDefinition, addSentence, unknownWordsOnly) unknownWordsOnly = int(unknownWordsOnly) result = [] textProc = TextProcessor(getDataLoader()) contents = textProc.do(userText, Settings.NoExcessiveReading(), True) for word, startPos, reading, definition, sentence in contents: isKnown = self.getWordStatus(word) if not unknownWordsOnly or not isKnown: result.append((word, reading, definition, sentence, isKnown)) logger.info('# of words to return: %d', len(result)) #contents = list(contents) # logging.info("%d records sent", len(result)) data = simplejson.dumps(result) h = HttpResponse(data, mimetype="application/json", status=200) return h
class TextProcessorTest2(unittest.TestCase): def setUp(self): self.textProc = TextProcessor(getDataLoader()) self.maxDiff = None def testONReadingFailure(self): # no word 車 with sha reading res = self.textProc.stepOne('逃走車', Settings.NoExcessiveReading()) def test2(self): res = self.textProc.do('電燈', Settings.NoExcessiveReading(), True) res = list(res) res = res
def dryBurn(): from pkgutil import iter_modules a=iter_modules() while True: try: x=a.next() except: break print (x[1], '<br>') setupLogger() contents = '船が検疫所に着いたのは' textProc = TextProcessor(getDataLoader()) for word, reading, definition, sentence in textProc.do(contents, Settings.NoExcessiveReading(), True): line = text_type('{0:<10} {1:<10} {2:<10} {3}\n').format(word, reading, definition,sentence) line = line.strip('\n') print(line.encode('utf-8'))
def testNoKanji(self): self.assertFalse(TextProcessor.hasKanji('が')) self.assertTrue(TextProcessor.hasKanji('所')) self.assertFalse(TextProcessor.hasKanji('。'))
def setUp(self): self.textProc = TextProcessor(getDataLoader()) self.maxDiff = None
class TextProcessorTest(unittest.TestCase): def setUp(self): self.textProc = TextProcessor(getDataLoader()) self.maxDiff = None def testDumpAll(self): res = self.textProc.do('船が検疫所に着いたのは', Settings.Verbose()) res = list(res) expected = \ [ ('船', 0, 'ふね', '(n,n-suf,ctr) ship/boat/watercraft/vessel/steamship/tank/tub/vat/trough/counter for boat-shaped containers (e.g. of sashimi)/(P)', '船が検疫所に着いたのは'), ('が', 1, 'が', '(prt,conj) indicates sentence subject (occasionally object)/indicates possessive (esp. in literary expressions)/but/however/still/and/(P)', '船が検疫所に着いたのは'), ('検疫', 2, 'けんえき', '(n,vs) quarantine/medical inspection', '船が検疫所に着いたのは'), ('所', 4, 'しょ', "(suf,ctr) counter for places", '船が検疫所に着いたのは'), ('に', 5, 'に', '(prt) indicates such things as location of person or thing, location of short-term action, etc./(P)', '船が検疫所に着いたのは'), ('着く', 6, 'つく', '(v5k) to arrive at/to reach/to sit on/to sit at (e.g. the table)/(P)', '船が検疫所に着いたのは'), ('た', 8, 'た', '(aux-v) indicate past completed or action/indicates light imperative', '船が検疫所に着いたのは'), ('の', 9, 'の', '(prt,fem) indicates possessive/nominalizes verbs and adjectives/substitutes for "ga" in subordinate phrases/(at sentence-end, falling tone) indicates a confident conclusion/(P)', '船が検疫所に着いたのは'), ('は', 10, 'は', '(prt) topic marker particle/indicates contrast with another option (stated or unstated)/adds emphasis/(P)', '船が検疫所に着いたのは') ] # self.assertEqual(len(expected), len(res)) self.assertEqual(expected, res) def testNoKanji(self): self.assertFalse(TextProcessor.hasKanji('が')) self.assertTrue(TextProcessor.hasKanji('所')) self.assertFalse(TextProcessor.hasKanji('。')) def testNoExcessiveReading(self): res = self.textProc.do('船が、検疫所に着いたのは', Settings.NoExcessiveReading()) res = list(res) expected = \ [ ('船', 0,'ふね', '(n,n-suf,ctr) ship/boat/watercraft/vessel/steamship/tank/tub/vat/trough/counter for boat-shaped containers (e.g. of sashimi)/(P)', '船が、検疫所に着いたのは'), ('が', 1, '', '(prt,conj) indicates sentence subject (occasionally object)/indicates possessive (esp. in literary expressions)/but/however/still/and/(P)', '船が、検疫所に着いたのは'), ('検疫', 3, 'けんえき', '(n,vs) quarantine/medical inspection', '船が、検疫所に着いたのは'), ('所', 5,'しょ', "(suf,ctr) counter for places", '船が、検疫所に着いたのは'), ('に', 6, '', '(prt) indicates such things as location of person or thing, location of short-term action, etc./(P)', '船が、検疫所に着いたのは'), ('着く', 7, 'つく', '(v5k) to arrive at/to reach/to sit on/to sit at (e.g. the table)/(P)', '船が、検疫所に着いたのは'), ('た', 9, '', '(aux-v) indicate past completed or action/indicates light imperative', '船が、検疫所に着いたのは'), ('の', 10, '', '(prt,fem) indicates possessive/nominalizes verbs and adjectives/substitutes for "ga" in subordinate phrases/(at sentence-end, falling tone) indicates a confident conclusion/(P)', '船が、検疫所に着いたのは'), ('は', 11, '', '(prt) topic marker particle/indicates contrast with another option (stated or unstated)/adds emphasis/(P)', '船が、検疫所に着いたのは') ] # self.assertEqual(len(expected), len(res)) self.assertEqual(expected, res) def testUnknownWord(self): res = self.textProc.do('デッキに昇って行った') res = list(res) self.assertEqual(6, len(res)) self.assertEqual('デッキ', res[0][0]) # for word, reading, definition, sentence in res: # print(word, reading ,definition) def testGetContext(self): snt = 'デッキに昇って行った' res = self.textProc.getContext(snt, WordInfo('デ', 0, '', 0, '') ) self.assertEqual(snt, res) def testExtentionOfUnknownToken(self): res = self.textProc.do('ジーン・モーラの姿は見えなかった。') res = list(res) self.assertEqual(7, len(list(res))) self.assertEqual('ジーン・モーラ', res[0][0]) def testONReading(self): res = self.textProc.do('数時間が') res = list(res) self.assertEqual(3, len(list(res))) self.assertEqual('すう', res[0][2]) def testMergeTwoNouns(self): res = self.textProc.do('検疫所に', Settings.NoExcessiveReading(), True) res = list(res) self.assertEqual(2, len(list(res))) self.assertEqual(('検疫所', 0, 'けんえきじょ'), res[0][0:3]) self.assertEqual('(n) quarantine station', res[0][3]) self.assertEqual('に', res[1][0]) def testMergeTwoVerbs(self): res = self.textProc.do('頃にちがいない。', Settings.NoExcessiveReading(), True) res = list(res) self.assertEqual(3, len(list(res))) self.assertEqual(('ちがいない', 2, ''), res[2][0:3]) self.assertEqual('(exp,adj-i) (phrase) sure/no mistaking it/for certain/(P)', res[2][3]) def testMergeTwoNumerics(self): res = self.textProc.do('が一匹と', Settings.NoExcessiveReading(), True) res = list(res) self.assertEqual(3, len(list(res))) self.assertEqual(('一匹', 1, 'いっぴき'), res[1][0:3]) self.assertEqual('(n,arch) one animal (small)/two-tan bolt of cloth', res[1][3]) def testPrefixMerger(self): res = self.textProc.do('準強姦の罪に', Settings.NoExcessiveReading(), True) res = list(res) self.assertEqual(('準強姦', 0, 'じゅんごうかん'), res[0][0:3]) def testThreeTokenJoin(self): result = self.textProc.do('海泡石', Settings.NoExcessiveReading(), True) result = list(map(operator.itemgetter(0), result)) self.assertEquals(['海泡石'], result)