class TokenCountFeatureExtractorTests(unittest.TestCase): def setUp(self): module_path = os.path.dirname(__file__) self.module_path = module_path self.tokenCounter = TokenCountFeatureExtractor() def test_get_features(self): # { 'token': <token>, index: <idx>, 'source': [<source toks>]', 'target': [<target toks>], 'tag': <tag>} vector = self.tokenCounter.get_features( {'token':'a', 'index':2, 'source':[u'c',u'\'',u'est',u'un',u'garçon'], 'target':[u'this',u'is',u'a',u'boy',u'.'], 'tag':'G'}) # the tokenCounter outputs three features self.assertEqual(len(vector), 3) self.assertEqual(vector[0], 5.0) self.assertEqual(vector[1], 5.0) self.assertEqual(vector[2], 1.0)
class TokenCountFeatureExtractorTests(unittest.TestCase): def setUp(self): module_path = os.path.dirname(__file__) self.module_path = module_path self.tokenCounter = TokenCountFeatureExtractor() def test_get_features(self): # { 'token': <token>, index: <idx>, 'source': [<source toks>]', 'target': [<target toks>], 'tag': <tag>} vector = self.tokenCounter.get_features({ 'token': 'a', 'index': 2, 'source': [u'c', u'\'', u'est', u'un', u'garçon'], 'target': [u'this', u'is', u'a', u'boy', u'.'], 'tag': 'G' }) # the tokenCounter outputs three features self.assertEqual(len(vector), 3) self.assertEqual(vector[0], 5.0) self.assertEqual(vector[1], 5.0) self.assertEqual(vector[2], 1.0)
def setUp(self): module_path = os.path.dirname(__file__) self.module_path = module_path self.tokenCounter = TokenCountFeatureExtractor()