Beispiel #1
0
class HDClustering(object):

    def __init__(self, cfg, trace_every=0,
                 get_body=None, get_label=None, get_prefix=None, seed=None,
                 min_support=None, normalizer=None, tokenizer=None):

        """Read configuration"""
        self.cfg = cfg
        self._get_body = get_body
        self._get_label = get_label
        self._get_prefix = get_prefix

        self.trace_every = trace_every

        # Set options
        self.random_state = cfg['random_state'] if seed is None else seed
        cfg_signer = cfg['signer']
        self.min_support = cfg_signer['min_support'] if min_support is None else min_support

        # tokenizer
        self.tokenizer = RegexTokenizer() if tokenizer is None else tokenizer

        # Configure minhash signer
        sig_width = cfg_signer['width']
        lsh_hasher = LSHC(width=sig_width, seed=self.random_state, **cfg_signer['lsh'])
        self.signer = MinHashSignature(sig_width,
                                       lsh_hasher=lsh_hasher,
                                       universe_size=cfg_signer['universe_size'],
                                       kmin=cfg_signer['kmin'],
                                       seed=self.random_state)

        # Configure shingler
        cfg_key_shingle = cfg['shingler']
        self.shingler = get_default_shingler(**cfg_key_shingle)

        # Configure sketch comparison algorithm
        self.sketch_enabled = False
        self.cluster_builder = Cluster(min_support=self.min_support)

    def _map_iter(self, data):
        """Find clusters in an iterable"""

        get_body = self._get_body
        get_label = self._get_label
        get_prefix = self._get_prefix

        for i, obj in enumerate(data):
            body = obj if get_body is None else get_body(obj)
            label = i if get_label is None else get_label(obj)
            prefix = None if get_prefix is None else get_prefix(obj)

            for feat in self._map_item(obj, body, label, prefix):
                yield feat

    def _map_item(self, obj, body, label, prefix=None):

        # Extract features
        obj_content = obj['content']
        normalized_content, meta = self.normalizer.normalize(obj_content)
        content_tokens = self.tokenizer.tokenize(normalized_content)

        features = self.shingler.get_shingles(content_tokens, prefix=prefix)
        keys = self.signer.get_signature(features)
        sketch = None
        yield (keys, (label, sketch))

    def clusters_from_iter(self, data):
        """Find clusters in an iterable"""

        cluster_builder = self.cluster_builder
        trace_every = self.trace_every
        for i, obj in enumerate(self._map_iter(data)):
            if trace_every > 0 and (not i % trace_every):
                LOG.info("Processing line " + str(i))

            keys, val = obj
            label, sketch = val \
                if isinstance(val, tuple) \
                else (val, None)
            cluster_builder.add_item(keys, label=label, sketch=sketch)

        return cluster_builder.get_clusters()

    def mapper(self, obj):
        """Perform a mapper task in MR"""
        get_body = self._get_body
        get_label = self._get_label
        get_prefix = self._get_prefix

        body = obj if get_body is None else get_body(obj)
        label = obj if get_label is None else get_label(obj)
        prefix = None if get_prefix is None else get_prefix(obj)

        for keys, val in self._map_item(obj, body, label, prefix):
            for key in keys:
                yield key, val

    def reducer(self, key, tuple_gen):
        """Perform a reducer task in MR

        If sketches enabled, data consists of::

            (key, [(lbl, sk), (lbl, sk), (lbl, sk)])

        Otherwise::

            (key, [lbl, lbl, lbl])

        """

        # If not using sketches, we are done
        return key, list(set(tuple_gen))
Beispiel #2
0
class HDClustering(object):

    def __init__(self, cfg, trace_every=0,
                 content_field='content',
                 get_body=None, get_label=None, get_prefix=None, min_support=None,
                 seed=0, tokenizer=None):

        """Read configuration"""
        self.cfg = cfg
        self._get_body = get_body
        self._get_label = get_label
        self._get_prefix = get_prefix

        self.trace_every = trace_every
        self.get_content = operator.itemgetter(content_field)

        # Set options
        self.min_support = cfg['min_support'] if min_support is None else min_support

        # Tokenizer
        self.tokenizer = RegexTokenizer() if tokenizer is None else tokenizer

        # Configure minhash signer
        sig_width = cfg['sig_width']
        lsh_hasher = LSHC(width=sig_width, **cfg['lsh_options'])
        self.signer = MinHashSignature(sig_width,
                                       lsh_hasher=lsh_hasher,
                                       kmin=cfg['kmin'])

        # Configure shingler
        cfg_key_shingle = cfg['shingler']
        self.shingler = get_default_shingler(**cfg_key_shingle)

        # Configure sketch comparison algorithm
        cfg_sketch = cfg['sketch']
        self.sketch_enabled = cfg_sketch['enabled']
        self.sketch_dist_fn = None
        self.max_dist = None
        if self.sketch_enabled:
            algorithm_name = cfg_sketch['algorithm']
            try:
                sketch_algorithm = getattr(SketchModel, algorithm_name)
            except AttributeError:
                raise RuntimeError("Unknown sketch model specified: '%s'"
                                   % algorithm_name)
            self.sketch_bits = cfg_sketch['size']
            cfg_sketch_shingler = cfg_sketch['shingler']
            if not cfg_sketch_shingler['enabled']:
                # if sketch shingler is disabled, we also disable signer
                # as we will use default signer
                self.sketch_shingler = None
                self.sketch_signer = None
            elif sketch_algorithm == SketchModel.simhash:
                del cfg_sketch_shingler['enabled']
                self.sketch_shingler = Shingler(**cfg_sketch_shingler)
                self.sketch_signer = SimHashSignature(self.sketch_bits, seed=seed)
            elif sketch_algorithm == SketchModel.minhash:
                del cfg_sketch_shingler['enabled']
                self.sketch_shingler = Shingler(**cfg_sketch_shingler)
                self.sketch_signer = MinHashSketchSignature(self.sketch_bits, seed=seed)

            self.sketch_shingler._tokenizer = None

            self.max_dist = \
                int(floor(self.sketch_bits *
                          (1.0 - float(cfg_sketch['resemblance']))))
            self.sketch_dist_fn = hamming
            self.sketch_operator = OPERATOR_MAP[cfg_sketch.get('operator', 'and')]
        self.cluster_builder = Cluster(sketch_dist_fn=self.sketch_dist_fn,
                                       max_dist=self.max_dist,
                                       min_support=self.min_support,
                                       sketch_operator=self.sketch_operator)

    def _map_iter(self, data):
        """Find clusters in an iterable"""

        get_body = self._get_body
        get_label = self._get_label
        get_prefix = self._get_prefix

        for i, obj in enumerate(data):
            body = obj if get_body is None else get_body(obj)
            label = i if get_label is None else get_label(obj)
            prefix = None if get_prefix is None else get_prefix(obj)

            for feat in self._map_item(obj, body, label, prefix):
                yield feat

    def _map_item(self, obj, body, label, prefix=None):

        # Extract features
        obj_content = self.get_content(obj)
        content_tokens = self.tokenizer.tokenize(obj_content)

        features = self.shingler.get_shingles(content_tokens, prefix=prefix)
        if self.sketch_enabled and (self.sketch_shingler is None or self.sketch_signer is None):
            keys, sketch = self.signer.get_signature(features, with_sketch=True)
        elif self.sketch_enabled and (self.sketch_shingler is not None and self.sketch_signer is not None):
            keys = self.signer.get_signature(features)
            sketch_features = self.sketch_shingler.get_shingles(content_tokens)
            sketch = self.sketch_signer.get_signature(sketch_features)
        else:
            keys = self.signer.get_signature(features)
            sketch = None
        yield (keys, (label, sketch))

    def clusters_from_iter(self, data):
        """Find clusters in an iterable"""

        cluster_builder = self.cluster_builder
        trace_every = self.trace_every
        for i, obj in enumerate(self._map_iter(data)):
            if trace_every > 0 and (not i % trace_every):
                LOG.info("Processing line " + str(i))

            keys, val = obj
            label, sketch = val \
                if isinstance(val, tuple) \
                else (val, None)
            cluster_builder.add_item(keys, label=label, sketch=sketch)

        return cluster_builder.get_clusters()

    def mapper(self, obj):
        """Perform a mapper task in MR"""
        get_body = self._get_body
        get_label = self._get_label
        get_prefix = self._get_prefix

        body = obj if get_body is None else get_body(obj)
        label = obj if get_label is None else get_label(obj)
        prefix = None if get_prefix is None else get_prefix(obj)

        for keys, val in self._map_item(obj, body, label, prefix):
            for key in keys:
                yield key, val

    def reducer(self, key, tuple_gen):
        """Perform a reducer task in MR

        If sketches enabled, data consists of::

            (key, [(lbl, sk), (lbl, sk), (lbl, sk)])

        Otherwise::

            (key, [lbl, lbl, lbl])
        """

        # If not using sketches, we are done
        if self.sketch_dist_fn is None:
            return key, list(set(tuple_gen))

        # create a dict mappipng a label to a sketch
        return key, dict(tuple_gen).items()