def test_minhash_sketch_similarity(self):
"""Signatures should be consistent"""
sh = MinHashSketchSignature(64)
sig1 = sh.get_signature("abracadabra")
sig2 = sh.get_signature("abracadabra")
dist = hamming(sig1, sig2)
self.assertEqual(dist, 0)
sig1 = sh.get_signature("abracadabra")
sig2 = sh.get_signature("arbcd")
dist = hamming(sig1, sig2)
self.assertEqual(dist, 0)
sig1 = sh.get_signature("abracadabra")
sig2 = sh.get_signature("")
dist = hamming(sig1, sig2)
self.assertEqual(dist, 26)
def __init__(self,
cfg,
content_filter=None,
trace_every=0,
get_body=None,
get_label=None,
get_prefix=None,
min_support=None,
seed=0,
normalizer=None,
tokenizer=None):
"""Read configuration"""
self.cfg = cfg
self._get_body = get_body
self._get_label = get_label
self._get_prefix = get_prefix
self.trace_every = trace_every
# Set options
self.content_filter = content_filter
self.min_support = cfg[
'min_support'] if min_support is None else min_support
# normalizer and tokenizer
self.normalizer = get_default_normalizer(
**cfg.get('preprocessor', {}).get('normalizer', {})) \
if normalizer is None else normalizer
self.tokenizer = get_default_tokenizer(
) if tokenizer is None else tokenizer
# Configure minhash signer
sig_width = cfg['sig_width']
lsh_hasher = LSHC(width=sig_width, **cfg['lsh_options'])
self.signer = MinHashSignature(sig_width,
lsh_hasher=lsh_hasher,
kmin=cfg['kmin'])
# Configure shingler
cfg_key_shingle = cfg['shingler']
self.shingler = get_default_shingler(**cfg_key_shingle)
# Configure sketch comparison algorithm
cfg_sketch = cfg['sketch']
self.sketch_enabled = cfg_sketch['enabled']
self.sketch_dist_fn = None
self.max_dist = None
if self.sketch_enabled:
algorithm_name = cfg_sketch['algorithm']
try:
sketch_algorithm = getattr(SketchModel, algorithm_name)
except AttributeError:
raise RuntimeError("Unknown sketch model specified: '%s'" %
algorithm_name)
self.sketch_bits = cfg_sketch['size']
cfg_sketch_shingler = cfg_sketch['shingler']
if not cfg_sketch_shingler['enabled']:
# if sketch shingler is disabled, we also disable signer
# as we will use default signer
self.sketch_shingler = None
self.sketch_signer = None
elif sketch_algorithm == SketchModel.simhash:
del cfg_sketch_shingler['enabled']
self.sketch_shingler = Shingler(**cfg_sketch_shingler)
self.sketch_signer = SimHashSignature(self.sketch_bits,
seed=seed)
elif sketch_algorithm == SketchModel.minhash:
del cfg_sketch_shingler['enabled']
self.sketch_shingler = Shingler(**cfg_sketch_shingler)
self.sketch_signer = MinHashSketchSignature(self.sketch_bits,
seed=seed)
self.sketch_shingler._tokenizer = None
self.sketch_shingler._normalizer = None
self.max_dist = \
int(floor(self.sketch_bits *
(1.0 - float(cfg_sketch['resemblance']))))
self.sketch_dist_fn = hamming
self.sketch_operator = OPERATOR_MAP[cfg_sketch.get(
'operator', 'and')]
self.cluster_builder = Cluster(sketch_dist_fn=self.sketch_dist_fn,
max_dist=self.max_dist,
min_support=self.min_support,
sketch_operator=self.sketch_operator)
