def __init__(self, token2index=None, token_parser=None):
"""
:param token2index: The mapping from tokens to bag keys. If None, no mapping is performed.
:param token_parser: Specify token parser if you want to use a custom one. \
:class:'NoopTokenParser' is used if it is not specified.
"""
self._token2index = FakeVocabulary() if token2index is None else token2index
self._token_parser = NoopTokenParser() if token_parser is None else token_parser
def __init__(self,
split_stem=False,
type="tree",
max_distance=DEFAULT_MAX_DISTANCE,
**kwargs):
super().__init__(**kwargs)
Uast2IdDistance = self.DistanceType.resolve(type)
self.uast2id_distance = Uast2IdDistance(
token_parser=NoopTokenParser() if not split_stem else None,
max_distance=max_distance)
def __init__(self, split_stem=False, **kwargs):
super().__init__(**kwargs)
self.uast2id_sequence = Uast2IdSequence(
None,
NoopTokenParser() if not split_stem else None)
def setUp(self):
self.uast2id_sequence = Uast2IdSequence(token_parser=NoopTokenParser())
self.uast = BblfshClient("0.0.0.0:9432").parse(SOURCE_PY).uast
class RoleIdsExtractor(Extractor):
NAME = "roleids"
ALGORITHM = Uast2RoleIdPairs(token_parser=NoopTokenParser())
def setUp(self):
self.tp = NoopTokenParser()
def setUp(self):
self.uast2role_id_pairs = Uast2IdTreeDistance(token_parser=NoopTokenParser(),
max_distance=4)
self.uast = BblfshClient("0.0.0.0:9432").parse(SOURCE_PY).uast
self.maxDiff = None
def setUp(self):
self.uast2role_id_pairs = Uast2RoleIdPairs(token_parser=NoopTokenParser())
self.uast = BblfshClient("0.0.0.0:9432").parse(SOURCE_PY).uast
def __init__(self, docfreq_threshold=None, split_stem=True, **kwargs):
super().__init__(docfreq_threshold, **kwargs)
self.id2bag = UastIds2Bag(
None,
NoopTokenParser() if not split_stem else None)
def __init__(self, split_stem=False, **kwargs):
super().__init__(**kwargs)
self.uast2role_id_pair = Uast2RoleIdPairs(
None,
NoopTokenParser() if not split_stem else None)