class TokenCountFeatureExtractorTests(unittest.TestCase):
def setUp(self):
module_path = os.path.dirname(__file__)
self.module_path = module_path
self.tokenCounter = TokenCountFeatureExtractor()
def test_get_features(self):
# { 'token': <token>, index: <idx>, 'source': [<source toks>]', 'target': [<target toks>], 'tag': <tag>}
vector = self.tokenCounter.get_features( {'token':'a', 'index':2, 'source':[u'c',u'\'',u'est',u'un',u'garçon'], 'target':[u'this',u'is',u'a',u'boy',u'.'], 'tag':'G'})
# the tokenCounter outputs three features
self.assertEqual(len(vector), 3)
self.assertEqual(vector[0], 5.0)
self.assertEqual(vector[1], 5.0)
self.assertEqual(vector[2], 1.0)
class TokenCountFeatureExtractorTests(unittest.TestCase):
def setUp(self):
module_path = os.path.dirname(__file__)
self.module_path = module_path
self.tokenCounter = TokenCountFeatureExtractor()
def test_get_features(self):
# { 'token': <token>, index: <idx>, 'source': [<source toks>]', 'target': [<target toks>], 'tag': <tag>}
vector = self.tokenCounter.get_features({
'token':
'a',
'index':
2,
'source': [u'c', u'\'', u'est', u'un', u'garçon'],
'target': [u'this', u'is', u'a', u'boy', u'.'],
'tag':
'G'
})
# the tokenCounter outputs three features
self.assertEqual(len(vector), 3)
self.assertEqual(vector[0], 5.0)
self.assertEqual(vector[1], 5.0)
self.assertEqual(vector[2], 1.0)
def setUp(self):
module_path = os.path.dirname(__file__)
self.module_path = module_path
self.tokenCounter = TokenCountFeatureExtractor()
def setUp(self):
module_path = os.path.dirname(__file__)
self.module_path = module_path
self.tokenCounter = TokenCountFeatureExtractor()
